infrahub-server 1.2.0rc0__py3-none-any.whl → 1.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- infrahub/api/dependencies.py +6 -6
- infrahub/api/diff/validation_models.py +7 -7
- infrahub/api/schema.py +1 -1
- infrahub/artifacts/models.py +5 -3
- infrahub/artifacts/tasks.py +3 -5
- infrahub/cli/__init__.py +13 -9
- infrahub/cli/constants.py +3 -0
- infrahub/cli/db.py +165 -183
- infrahub/cli/upgrade.py +146 -0
- infrahub/computed_attribute/gather.py +185 -0
- infrahub/computed_attribute/models.py +240 -12
- infrahub/computed_attribute/tasks.py +72 -432
- infrahub/computed_attribute/triggers.py +13 -47
- infrahub/config.py +43 -32
- infrahub/context.py +14 -0
- infrahub/core/account.py +4 -4
- infrahub/core/attribute.py +58 -58
- infrahub/core/branch/tasks.py +74 -22
- infrahub/core/changelog/diff.py +95 -36
- infrahub/core/changelog/models.py +217 -43
- infrahub/core/constants/__init__.py +28 -0
- infrahub/core/constants/infrahubkind.py +2 -0
- infrahub/core/constants/schema.py +2 -0
- infrahub/core/constraint/node/runner.py +9 -8
- infrahub/core/diff/branch_differ.py +10 -10
- infrahub/core/diff/enricher/cardinality_one.py +5 -0
- infrahub/core/diff/enricher/hierarchy.py +17 -4
- infrahub/core/diff/enricher/labels.py +5 -0
- infrahub/core/diff/enricher/path_identifier.py +4 -0
- infrahub/core/diff/ipam_diff_parser.py +4 -5
- infrahub/core/diff/model/diff.py +27 -27
- infrahub/core/diff/model/path.py +32 -9
- infrahub/core/diff/parent_node_adder.py +78 -0
- infrahub/core/diff/payload_builder.py +13 -2
- infrahub/core/diff/query/filters.py +2 -2
- infrahub/core/diff/query/merge.py +20 -17
- infrahub/core/diff/query/save.py +188 -182
- infrahub/core/diff/query/summary_counts_enricher.py +51 -4
- infrahub/core/diff/query_parser.py +4 -4
- infrahub/core/diff/repository/deserializer.py +8 -3
- infrahub/core/diff/repository/repository.py +156 -38
- infrahub/core/diff/tasks.py +4 -4
- infrahub/core/graph/__init__.py +1 -1
- infrahub/core/graph/index.py +3 -0
- infrahub/core/initialization.py +1 -10
- infrahub/core/ipam/constants.py +3 -4
- infrahub/core/ipam/reconciler.py +12 -12
- infrahub/core/ipam/utilization.py +10 -13
- infrahub/core/manager.py +36 -36
- infrahub/core/merge.py +7 -7
- infrahub/core/migrations/__init__.py +2 -3
- infrahub/core/migrations/graph/__init__.py +12 -3
- infrahub/core/migrations/graph/m017_add_core_profile.py +1 -5
- infrahub/core/migrations/graph/m018_uniqueness_nulls.py +4 -4
- infrahub/core/migrations/graph/m019_restore_rels_to_time.py +256 -0
- infrahub/core/migrations/graph/m020_duplicate_edges.py +160 -0
- infrahub/core/migrations/graph/m021_missing_hierarchy_merge.py +51 -0
- infrahub/core/migrations/graph/m022_add_generate_template_attr.py +48 -0
- infrahub/core/migrations/graph/m023_deduplicate_cardinality_one_relationships.py +96 -0
- infrahub/core/migrations/query/attribute_add.py +2 -2
- infrahub/core/migrations/query/node_duplicate.py +43 -26
- infrahub/core/migrations/query/schema_attribute_update.py +2 -2
- infrahub/core/migrations/schema/models.py +19 -4
- infrahub/core/migrations/schema/node_remove.py +26 -12
- infrahub/core/migrations/schema/tasks.py +2 -2
- infrahub/core/migrations/shared.py +16 -16
- infrahub/core/models.py +15 -6
- infrahub/core/node/__init__.py +43 -39
- infrahub/core/node/base.py +2 -4
- infrahub/core/node/constraints/attribute_uniqueness.py +2 -2
- infrahub/core/node/constraints/grouped_uniqueness.py +99 -47
- infrahub/core/node/constraints/interface.py +1 -2
- infrahub/core/node/delete_validator.py +3 -5
- infrahub/core/node/ipam.py +4 -4
- infrahub/core/node/permissions.py +7 -7
- infrahub/core/node/resource_manager/ip_address_pool.py +6 -6
- infrahub/core/node/resource_manager/ip_prefix_pool.py +6 -6
- infrahub/core/node/resource_manager/number_pool.py +3 -3
- infrahub/core/path.py +12 -12
- infrahub/core/property.py +11 -11
- infrahub/core/protocols.py +7 -0
- infrahub/core/protocols_base.py +21 -21
- infrahub/core/query/__init__.py +33 -33
- infrahub/core/query/attribute.py +6 -4
- infrahub/core/query/diff.py +3 -3
- infrahub/core/query/node.py +98 -37
- infrahub/core/query/relationship.py +228 -40
- infrahub/core/query/resource_manager.py +2 -0
- infrahub/core/query/standard_node.py +3 -3
- infrahub/core/query/subquery.py +9 -9
- infrahub/core/registry.py +13 -15
- infrahub/core/relationship/constraints/count.py +3 -4
- infrahub/core/relationship/constraints/peer_kind.py +3 -4
- infrahub/core/relationship/constraints/profiles_kind.py +2 -2
- infrahub/core/relationship/model.py +51 -59
- infrahub/core/schema/attribute_schema.py +16 -8
- infrahub/core/schema/basenode_schema.py +105 -44
- infrahub/core/schema/computed_attribute.py +3 -3
- infrahub/core/schema/definitions/core/__init__.py +147 -0
- infrahub/core/schema/definitions/core/account.py +171 -0
- infrahub/core/schema/definitions/core/artifact.py +136 -0
- infrahub/core/schema/definitions/core/builtin.py +24 -0
- infrahub/core/schema/definitions/core/check.py +68 -0
- infrahub/core/schema/definitions/core/core.py +17 -0
- infrahub/core/schema/definitions/core/generator.py +100 -0
- infrahub/core/schema/definitions/core/graphql_query.py +79 -0
- infrahub/core/schema/definitions/core/group.py +108 -0
- infrahub/core/schema/definitions/core/ipam.py +193 -0
- infrahub/core/schema/definitions/core/lineage.py +19 -0
- infrahub/core/schema/definitions/core/menu.py +48 -0
- infrahub/core/schema/definitions/core/permission.py +163 -0
- infrahub/core/schema/definitions/core/profile.py +18 -0
- infrahub/core/schema/definitions/core/propose_change.py +97 -0
- infrahub/core/schema/definitions/core/propose_change_comment.py +193 -0
- infrahub/core/schema/definitions/core/propose_change_validator.py +328 -0
- infrahub/core/schema/definitions/core/repository.py +286 -0
- infrahub/core/schema/definitions/core/resource_pool.py +170 -0
- infrahub/core/schema/definitions/core/template.py +27 -0
- infrahub/core/schema/definitions/core/transform.py +96 -0
- infrahub/core/schema/definitions/core/webhook.py +134 -0
- infrahub/core/schema/definitions/internal.py +16 -16
- infrahub/core/schema/dropdown.py +3 -4
- infrahub/core/schema/generated/attribute_schema.py +15 -18
- infrahub/core/schema/generated/base_node_schema.py +12 -14
- infrahub/core/schema/generated/node_schema.py +3 -5
- infrahub/core/schema/generated/relationship_schema.py +9 -11
- infrahub/core/schema/generic_schema.py +2 -2
- infrahub/core/schema/manager.py +20 -9
- infrahub/core/schema/node_schema.py +4 -2
- infrahub/core/schema/relationship_schema.py +14 -6
- infrahub/core/schema/schema_branch.py +292 -144
- infrahub/core/schema/schema_branch_computed.py +41 -4
- infrahub/core/task/task.py +3 -3
- infrahub/core/task/user_task.py +15 -15
- infrahub/core/timestamp.py +3 -3
- infrahub/core/utils.py +20 -18
- infrahub/core/validators/__init__.py +1 -3
- infrahub/core/validators/aggregated_checker.py +2 -2
- infrahub/core/validators/attribute/choices.py +2 -2
- infrahub/core/validators/attribute/enum.py +2 -2
- infrahub/core/validators/attribute/kind.py +2 -2
- infrahub/core/validators/attribute/length.py +2 -2
- infrahub/core/validators/attribute/optional.py +2 -2
- infrahub/core/validators/attribute/regex.py +2 -2
- infrahub/core/validators/attribute/unique.py +2 -2
- infrahub/core/validators/checks_runner.py +25 -2
- infrahub/core/validators/determiner.py +1 -3
- infrahub/core/validators/interface.py +6 -2
- infrahub/core/validators/model.py +22 -3
- infrahub/core/validators/models/validate_migration.py +17 -4
- infrahub/core/validators/node/attribute.py +2 -2
- infrahub/core/validators/node/generate_profile.py +2 -2
- infrahub/core/validators/node/hierarchy.py +3 -5
- infrahub/core/validators/node/inherit_from.py +27 -5
- infrahub/core/validators/node/relationship.py +2 -2
- infrahub/core/validators/relationship/count.py +4 -4
- infrahub/core/validators/relationship/optional.py +2 -2
- infrahub/core/validators/relationship/peer.py +2 -2
- infrahub/core/validators/shared.py +2 -2
- infrahub/core/validators/tasks.py +8 -0
- infrahub/core/validators/uniqueness/checker.py +22 -21
- infrahub/core/validators/uniqueness/index.py +2 -2
- infrahub/core/validators/uniqueness/model.py +11 -11
- infrahub/database/__init__.py +27 -22
- infrahub/database/metrics.py +7 -1
- infrahub/dependencies/builder/constraint/grouped/node_runner.py +1 -3
- infrahub/dependencies/builder/diff/deserializer.py +3 -1
- infrahub/dependencies/builder/diff/enricher/hierarchy.py +3 -1
- infrahub/dependencies/builder/diff/parent_node_adder.py +8 -0
- infrahub/dependencies/component/registry.py +2 -2
- infrahub/events/__init__.py +25 -2
- infrahub/events/artifact_action.py +64 -0
- infrahub/events/branch_action.py +33 -22
- infrahub/events/generator.py +71 -0
- infrahub/events/group_action.py +51 -21
- infrahub/events/models.py +18 -19
- infrahub/events/node_action.py +88 -37
- infrahub/events/repository_action.py +5 -18
- infrahub/events/schema_action.py +4 -9
- infrahub/events/utils.py +16 -0
- infrahub/events/validator_action.py +55 -0
- infrahub/exceptions.py +62 -26
- infrahub/generators/models.py +2 -3
- infrahub/generators/tasks.py +24 -4
- infrahub/git/base.py +87 -36
- infrahub/git/integrator.py +48 -48
- infrahub/git/models.py +101 -9
- infrahub/git/repository.py +3 -3
- infrahub/git/tasks.py +408 -6
- infrahub/git/utils.py +48 -0
- infrahub/git/worktree.py +1 -2
- infrahub/git_credential/askpass.py +1 -2
- infrahub/graphql/analyzer.py +12 -0
- infrahub/graphql/app.py +13 -15
- infrahub/graphql/context.py +39 -0
- infrahub/graphql/initialization.py +3 -0
- infrahub/graphql/loaders/node.py +2 -12
- infrahub/graphql/loaders/peers.py +77 -0
- infrahub/graphql/loaders/shared.py +13 -0
- infrahub/graphql/manager.py +17 -19
- infrahub/graphql/mutations/artifact_definition.py +5 -5
- infrahub/graphql/mutations/branch.py +26 -1
- infrahub/graphql/mutations/computed_attribute.py +9 -5
- infrahub/graphql/mutations/diff.py +23 -11
- infrahub/graphql/mutations/diff_conflict.py +5 -0
- infrahub/graphql/mutations/generator.py +83 -0
- infrahub/graphql/mutations/graphql_query.py +5 -5
- infrahub/graphql/mutations/ipam.py +54 -74
- infrahub/graphql/mutations/main.py +195 -132
- infrahub/graphql/mutations/menu.py +7 -7
- infrahub/graphql/mutations/models.py +2 -4
- infrahub/graphql/mutations/node_getter/by_default_filter.py +10 -10
- infrahub/graphql/mutations/node_getter/by_hfid.py +1 -3
- infrahub/graphql/mutations/node_getter/by_id.py +1 -3
- infrahub/graphql/mutations/node_getter/interface.py +1 -2
- infrahub/graphql/mutations/proposed_change.py +7 -7
- infrahub/graphql/mutations/relationship.py +93 -19
- infrahub/graphql/mutations/repository.py +8 -8
- infrahub/graphql/mutations/resource_manager.py +3 -3
- infrahub/graphql/mutations/schema.py +19 -4
- infrahub/graphql/mutations/webhook.py +137 -0
- infrahub/graphql/parser.py +4 -4
- infrahub/graphql/permissions.py +1 -10
- infrahub/graphql/queries/diff/tree.py +19 -14
- infrahub/graphql/queries/event.py +5 -2
- infrahub/graphql/queries/ipam.py +2 -2
- infrahub/graphql/queries/relationship.py +2 -2
- infrahub/graphql/queries/search.py +2 -2
- infrahub/graphql/resolvers/many_relationship.py +264 -0
- infrahub/graphql/resolvers/resolver.py +13 -110
- infrahub/graphql/schema.py +2 -0
- infrahub/graphql/subscription/graphql_query.py +2 -0
- infrahub/graphql/types/context.py +12 -0
- infrahub/graphql/types/event.py +84 -17
- infrahub/graphql/types/node.py +2 -2
- infrahub/graphql/utils.py +2 -2
- infrahub/groups/ancestors.py +29 -0
- infrahub/groups/parsers.py +107 -0
- infrahub/lock.py +20 -20
- infrahub/menu/constants.py +0 -1
- infrahub/menu/generator.py +9 -21
- infrahub/menu/menu.py +17 -38
- infrahub/menu/models.py +117 -16
- infrahub/menu/repository.py +111 -0
- infrahub/menu/utils.py +5 -8
- infrahub/message_bus/__init__.py +11 -13
- infrahub/message_bus/messages/__init__.py +1 -21
- infrahub/message_bus/messages/check_generator_run.py +3 -3
- infrahub/message_bus/messages/finalize_validator_execution.py +3 -0
- infrahub/message_bus/messages/proposed_change/request_proposedchange_refreshartifacts.py +6 -0
- infrahub/message_bus/messages/request_generatordefinition_check.py +2 -0
- infrahub/message_bus/messages/send_echo_request.py +1 -1
- infrahub/message_bus/operations/__init__.py +1 -10
- infrahub/message_bus/operations/check/__init__.py +2 -2
- infrahub/message_bus/operations/check/generator.py +1 -0
- infrahub/message_bus/operations/event/__init__.py +2 -2
- infrahub/message_bus/operations/event/worker.py +0 -3
- infrahub/message_bus/operations/finalize/validator.py +51 -1
- infrahub/message_bus/operations/requests/__init__.py +0 -2
- infrahub/message_bus/operations/requests/generator_definition.py +21 -23
- infrahub/message_bus/operations/requests/proposed_change.py +14 -10
- infrahub/permissions/globals.py +15 -0
- infrahub/pools/number.py +2 -4
- infrahub/proposed_change/models.py +3 -0
- infrahub/proposed_change/tasks.py +58 -45
- infrahub/pytest_plugin.py +13 -10
- infrahub/server.py +2 -3
- infrahub/services/__init__.py +2 -2
- infrahub/services/adapters/cache/__init__.py +4 -6
- infrahub/services/adapters/cache/nats.py +4 -5
- infrahub/services/adapters/cache/redis.py +3 -7
- infrahub/services/adapters/event/__init__.py +1 -1
- infrahub/services/adapters/message_bus/__init__.py +3 -3
- infrahub/services/adapters/message_bus/local.py +2 -2
- infrahub/services/adapters/message_bus/nats.py +4 -4
- infrahub/services/adapters/message_bus/rabbitmq.py +4 -4
- infrahub/services/adapters/workflow/local.py +2 -2
- infrahub/services/component.py +5 -5
- infrahub/services/protocols.py +7 -7
- infrahub/services/scheduler.py +1 -3
- infrahub/task_manager/event.py +102 -9
- infrahub/task_manager/models.py +27 -7
- infrahub/tasks/artifact.py +7 -6
- infrahub/telemetry/__init__.py +0 -0
- infrahub/telemetry/constants.py +9 -0
- infrahub/telemetry/database.py +86 -0
- infrahub/telemetry/models.py +65 -0
- infrahub/telemetry/task_manager.py +77 -0
- infrahub/{tasks/telemetry.py → telemetry/tasks.py} +49 -56
- infrahub/telemetry/utils.py +11 -0
- infrahub/trace.py +4 -4
- infrahub/transformations/tasks.py +2 -2
- infrahub/trigger/catalogue.py +4 -6
- infrahub/trigger/constants.py +0 -8
- infrahub/trigger/models.py +54 -5
- infrahub/trigger/setup.py +90 -0
- infrahub/trigger/tasks.py +35 -83
- infrahub/utils.py +11 -1
- infrahub/validators/__init__.py +0 -0
- infrahub/validators/events.py +42 -0
- infrahub/validators/tasks.py +41 -0
- infrahub/webhook/gather.py +17 -0
- infrahub/webhook/models.py +176 -44
- infrahub/webhook/tasks.py +154 -155
- infrahub/webhook/triggers.py +31 -7
- infrahub/workers/infrahub_async.py +2 -2
- infrahub/workers/utils.py +2 -2
- infrahub/workflows/catalogue.py +86 -35
- infrahub/workflows/initialization.py +8 -2
- infrahub/workflows/models.py +27 -1
- infrahub/workflows/utils.py +15 -6
- infrahub_sdk/client.py +35 -8
- infrahub_sdk/config.py +3 -0
- infrahub_sdk/context.py +13 -0
- infrahub_sdk/ctl/branch.py +3 -2
- infrahub_sdk/ctl/cli_commands.py +5 -1
- infrahub_sdk/ctl/utils.py +0 -16
- infrahub_sdk/exceptions.py +12 -0
- infrahub_sdk/generator.py +4 -1
- infrahub_sdk/graphql.py +45 -13
- infrahub_sdk/node.py +73 -22
- infrahub_sdk/protocols.py +21 -8
- infrahub_sdk/protocols_base.py +32 -11
- infrahub_sdk/query_groups.py +6 -35
- infrahub_sdk/schema/__init__.py +55 -26
- infrahub_sdk/schema/main.py +8 -0
- infrahub_sdk/task/__init__.py +11 -0
- infrahub_sdk/task/constants.py +3 -0
- infrahub_sdk/task/exceptions.py +25 -0
- infrahub_sdk/task/manager.py +551 -0
- infrahub_sdk/task/models.py +74 -0
- infrahub_sdk/testing/schemas/animal.py +9 -0
- infrahub_sdk/timestamp.py +142 -33
- infrahub_sdk/utils.py +29 -1
- {infrahub_server-1.2.0rc0.dist-info → infrahub_server-1.2.2.dist-info}/METADATA +8 -6
- {infrahub_server-1.2.0rc0.dist-info → infrahub_server-1.2.2.dist-info}/RECORD +349 -293
- {infrahub_server-1.2.0rc0.dist-info → infrahub_server-1.2.2.dist-info}/entry_points.txt +1 -0
- infrahub_testcontainers/constants.py +2 -0
- infrahub_testcontainers/container.py +157 -12
- infrahub_testcontainers/docker-compose.test.yml +31 -6
- infrahub_testcontainers/helpers.py +18 -73
- infrahub_testcontainers/host.py +41 -0
- infrahub_testcontainers/measurements.py +93 -0
- infrahub_testcontainers/models.py +38 -0
- infrahub_testcontainers/performance_test.py +166 -0
- infrahub_testcontainers/plugin.py +136 -0
- infrahub_testcontainers/prometheus.yml +30 -0
- infrahub/core/schema/definitions/core.py +0 -2286
- infrahub/message_bus/messages/check_repository_checkdefinition.py +0 -20
- infrahub/message_bus/messages/check_repository_mergeconflicts.py +0 -16
- infrahub/message_bus/messages/check_repository_usercheck.py +0 -26
- infrahub/message_bus/messages/event_branch_create.py +0 -11
- infrahub/message_bus/messages/event_branch_delete.py +0 -11
- infrahub/message_bus/messages/event_branch_rebased.py +0 -9
- infrahub/message_bus/messages/event_node_mutated.py +0 -15
- infrahub/message_bus/messages/event_schema_update.py +0 -9
- infrahub/message_bus/messages/request_repository_checks.py +0 -12
- infrahub/message_bus/messages/request_repository_userchecks.py +0 -18
- infrahub/message_bus/operations/check/repository.py +0 -293
- infrahub/message_bus/operations/event/node.py +0 -20
- infrahub/message_bus/operations/event/schema.py +0 -17
- infrahub/message_bus/operations/requests/repository.py +0 -133
- infrahub/webhook/constants.py +0 -1
- {infrahub_server-1.2.0rc0.dist-info → infrahub_server-1.2.2.dist-info}/LICENSE.txt +0 -0
- {infrahub_server-1.2.0rc0.dist-info → infrahub_server-1.2.2.dist-info}/WHEEL +0 -0
infrahub/task_manager/event.py
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import asyncio
|
|
3
4
|
from typing import TYPE_CHECKING, Any
|
|
4
5
|
|
|
5
6
|
from prefect.client.orchestration import PrefectClient, get_client
|
|
6
7
|
from prefect.events.schemas.events import Event as PrefectEventModel
|
|
8
|
+
from prefect.exceptions import PrefectHTTPStatusError
|
|
7
9
|
from pydantic import BaseModel, Field, TypeAdapter
|
|
8
10
|
|
|
11
|
+
from infrahub.core.constants import GLOBAL_BRANCH_NAME
|
|
12
|
+
from infrahub.exceptions import ServiceUnavailableError
|
|
9
13
|
from infrahub.log import get_logger
|
|
10
14
|
from infrahub.utils import get_nested_dict
|
|
11
15
|
|
|
@@ -22,6 +26,8 @@ class PrefectEventData(PrefectEventModel):
|
|
|
22
26
|
continue
|
|
23
27
|
if "infrahub.resource.label" not in resource:
|
|
24
28
|
continue
|
|
29
|
+
if resource.get("infrahub.resource.label") == GLOBAL_BRANCH_NAME:
|
|
30
|
+
return None
|
|
25
31
|
return resource.get("infrahub.resource.label")
|
|
26
32
|
return None
|
|
27
33
|
|
|
@@ -86,11 +92,10 @@ class PrefectEventData(PrefectEventModel):
|
|
|
86
92
|
|
|
87
93
|
def _return_node_mutation(self) -> dict[str, Any]:
|
|
88
94
|
attributes = []
|
|
95
|
+
relationships = []
|
|
89
96
|
|
|
90
97
|
for resource in self.related:
|
|
91
|
-
if resource.
|
|
92
|
-
"infrahub.attribute.name"
|
|
93
|
-
):
|
|
98
|
+
if resource.role == "infrahub.node.attribute_update" and resource.get("infrahub.attribute.name"):
|
|
94
99
|
attributes.append(
|
|
95
100
|
{
|
|
96
101
|
"name": resource.get("infrahub.attribute.name", ""),
|
|
@@ -104,15 +109,91 @@ class PrefectEventData(PrefectEventModel):
|
|
|
104
109
|
"action": resource.get("infrahub.attribute.action", "unchanged"),
|
|
105
110
|
}
|
|
106
111
|
)
|
|
112
|
+
elif resource.role == "infrahub.node.relationship_update":
|
|
113
|
+
relationships.append(
|
|
114
|
+
{
|
|
115
|
+
"name": resource.get("infrahub.relationship.name"),
|
|
116
|
+
"action": resource.get("infrahub.relationship.peer_status"),
|
|
117
|
+
"peer": {
|
|
118
|
+
"id": resource.get("infrahub.relationship.peer_id"),
|
|
119
|
+
"kind": resource.get("infrahub.relationship.peer_kind"),
|
|
120
|
+
},
|
|
121
|
+
}
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
return {"attributes": attributes, "relationships": relationships}
|
|
125
|
+
|
|
126
|
+
def _get_branch_name_from_resource(self) -> str:
|
|
127
|
+
return self.resource.get("infrahub.branch.name") or ""
|
|
128
|
+
|
|
129
|
+
def _return_artifact_event(self) -> dict[str, Any]:
|
|
130
|
+
checksum = ""
|
|
131
|
+
checksum_previous: str | None = None
|
|
132
|
+
storage_id = ""
|
|
133
|
+
storage_id_previous: str | None = None
|
|
134
|
+
artifact_definition_id = ""
|
|
135
|
+
for resource in self.related:
|
|
136
|
+
if resource.role == "infrahub.artifact":
|
|
137
|
+
checksum = resource.get("infrahub.artifact.checksum") or ""
|
|
138
|
+
checksum_previous = resource.get("infrahub.artifact.checksum_previous")
|
|
139
|
+
storage_id = resource.get("infrahub.artifact.storage_id") or ""
|
|
140
|
+
storage_id_previous = resource.get("infrahub.artifact.storage_id_previous")
|
|
141
|
+
artifact_definition_id = resource.get("infrahub.artifact.artifact_definition_id") or ""
|
|
142
|
+
|
|
143
|
+
return {
|
|
144
|
+
"checksum": checksum,
|
|
145
|
+
"checksum_previous": checksum_previous,
|
|
146
|
+
"storage_id": storage_id,
|
|
147
|
+
"storage_id_previous": storage_id_previous,
|
|
148
|
+
"artifact_definition_id": artifact_definition_id,
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
def _return_branch_created(self) -> dict[str, Any]:
|
|
152
|
+
return {"created_branch": self._get_branch_name_from_resource()}
|
|
153
|
+
|
|
154
|
+
def _return_branch_deleted(self) -> dict[str, Any]:
|
|
155
|
+
return {"deleted_branch": self._get_branch_name_from_resource()}
|
|
107
156
|
|
|
108
|
-
|
|
157
|
+
def _return_branch_merged(self) -> dict[str, Any]:
|
|
158
|
+
return {"source_branch": self._get_branch_name_from_resource()}
|
|
159
|
+
|
|
160
|
+
def _return_branch_rebased(self) -> dict[str, Any]:
|
|
161
|
+
return {"rebased_branch": self._get_branch_name_from_resource()}
|
|
162
|
+
|
|
163
|
+
def _return_group_event(self) -> dict[str, Any]:
|
|
164
|
+
members = []
|
|
165
|
+
ancestors = []
|
|
166
|
+
|
|
167
|
+
for resource in self.related:
|
|
168
|
+
if resource.role == "infrahub.group.member" and resource.get("infrahub.node.kind"):
|
|
169
|
+
members.append({"id": resource.id, "kind": resource.get("infrahub.node.kind")})
|
|
170
|
+
elif resource.role == "infrahub.group.ancestor" and resource.get("infrahub.node.kind"):
|
|
171
|
+
ancestors.append({"id": resource.id, "kind": resource.get("infrahub.node.kind")})
|
|
172
|
+
|
|
173
|
+
return {"members": members, "ancestors": ancestors}
|
|
109
174
|
|
|
110
175
|
def _return_event_specifics(self) -> dict[str, Any]:
|
|
176
|
+
"""Return event specific data based on the type of event being processed"""
|
|
177
|
+
|
|
178
|
+
event_specifics = {}
|
|
179
|
+
|
|
111
180
|
match self.event:
|
|
181
|
+
case "infrahub.artifact.created" | "infrahub.artifact.updated":
|
|
182
|
+
event_specifics = self._return_artifact_event()
|
|
112
183
|
case "infrahub.node.created" | "infrahub.node.updated" | "infrahub.node.deleted":
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
184
|
+
event_specifics = self._return_node_mutation()
|
|
185
|
+
case "infrahub.branch.created":
|
|
186
|
+
event_specifics = self._return_branch_created()
|
|
187
|
+
case "infrahub.branch.deleted":
|
|
188
|
+
event_specifics = self._return_branch_deleted()
|
|
189
|
+
case "infrahub.branch.merged":
|
|
190
|
+
event_specifics = self._return_branch_merged()
|
|
191
|
+
case "infrahub.branch.rebased":
|
|
192
|
+
event_specifics = self._return_branch_rebased()
|
|
193
|
+
case "infrahub.group.member_added" | "infrahub.group.member_removed":
|
|
194
|
+
event_specifics = self._return_group_event()
|
|
195
|
+
|
|
196
|
+
return event_specifics
|
|
116
197
|
|
|
117
198
|
def to_graphql(self) -> dict[str, Any]:
|
|
118
199
|
response = {
|
|
@@ -148,8 +229,20 @@ class PrefectEvent:
|
|
|
148
229
|
) -> PrefectEventResponse:
|
|
149
230
|
body = {"limit": limit, "filter": filters.model_dump(mode="json", exclude_none=True), "offset": offset}
|
|
150
231
|
|
|
151
|
-
|
|
152
|
-
|
|
232
|
+
# Retry due to https://github.com/PrefectHQ/prefect/issues/16299
|
|
233
|
+
for _ in range(1, 5):
|
|
234
|
+
prefect_error: PrefectHTTPStatusError | None = None
|
|
235
|
+
try:
|
|
236
|
+
response = await client._client.post("/infrahub/events/filter", json=body)
|
|
237
|
+
break
|
|
238
|
+
except PrefectHTTPStatusError as exc:
|
|
239
|
+
prefect_error = exc
|
|
240
|
+
await asyncio.sleep(0.1)
|
|
241
|
+
|
|
242
|
+
if prefect_error:
|
|
243
|
+
raise ServiceUnavailableError(
|
|
244
|
+
message=f"Unable to query prefect due to invalid response from the server (status_code={prefect_error.response.status_code})"
|
|
245
|
+
)
|
|
153
246
|
data: dict[str, Any] = response.json()
|
|
154
247
|
|
|
155
248
|
return PrefectEventResponse(
|
infrahub/task_manager/models.py
CHANGED
|
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import uuid
|
|
4
4
|
from collections import defaultdict
|
|
5
|
-
from typing import TYPE_CHECKING
|
|
5
|
+
from typing import TYPE_CHECKING, Any
|
|
6
6
|
from uuid import UUID
|
|
7
7
|
|
|
8
8
|
from prefect.client.schemas.objects import Log as PrefectLog # noqa: TC002
|
|
@@ -31,7 +31,7 @@ class RelatedNodeInfo(BaseModel):
|
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
class RelatedNodesInfo(BaseModel):
|
|
34
|
-
flows: dict[UUID, dict[str, RelatedNodeInfo]] = Field(default_factory=lambda: defaultdict(dict))
|
|
34
|
+
flows: dict[UUID, dict[str, RelatedNodeInfo]] = Field(default_factory=lambda: defaultdict(dict)) # type: ignore[arg-type]
|
|
35
35
|
nodes: dict[str, RelatedNodeInfo] = Field(default_factory=dict)
|
|
36
36
|
|
|
37
37
|
def add_nodes(self, flow_id: UUID, node_ids: list[str]) -> None:
|
|
@@ -64,7 +64,7 @@ class RelatedNodesInfo(BaseModel):
|
|
|
64
64
|
|
|
65
65
|
|
|
66
66
|
class FlowLogs(BaseModel):
|
|
67
|
-
logs: defaultdict[UUID, list[PrefectLog]] = Field(default_factory=lambda: defaultdict(list))
|
|
67
|
+
logs: defaultdict[UUID, list[PrefectLog]] = Field(default_factory=lambda: defaultdict(list)) # type: ignore[arg-type]
|
|
68
68
|
|
|
69
69
|
def to_graphql(self, flow_id: UUID) -> list[dict]:
|
|
70
70
|
return [
|
|
@@ -122,7 +122,26 @@ class InfrahubEventFilter(EventFilter):
|
|
|
122
122
|
if ids:
|
|
123
123
|
self.id = EventIDFilter(id=[uuid.UUID(id) for id in ids])
|
|
124
124
|
|
|
125
|
-
def add_event_type_filter(
|
|
125
|
+
def add_event_type_filter(
|
|
126
|
+
self, event_type: list[str] | None = None, event_type_filter: dict[str, Any] | None = None
|
|
127
|
+
) -> None:
|
|
128
|
+
event_type = event_type or []
|
|
129
|
+
event_type_filter = event_type_filter or {}
|
|
130
|
+
|
|
131
|
+
if branch_merged := event_type_filter.get("branch_merged"):
|
|
132
|
+
branches: list[str] = branch_merged.get("branches") or []
|
|
133
|
+
if "infrahub.branch.created" not in event_type:
|
|
134
|
+
event_type.append("infrahub.branch.merged")
|
|
135
|
+
if branches:
|
|
136
|
+
self.resource = EventResourceFilter(labels=ResourceSpecification({"infrahub.branch.name": branches}))
|
|
137
|
+
|
|
138
|
+
if branch_rebased := event_type_filter.get("branch_rebased"):
|
|
139
|
+
branches = branch_rebased.get("branches") or []
|
|
140
|
+
if "infrahub.branch.created" not in event_type:
|
|
141
|
+
event_type.append("infrahub.branch.rebased")
|
|
142
|
+
if branches:
|
|
143
|
+
self.resource = EventResourceFilter(labels=ResourceSpecification({"infrahub.branch.name": branches}))
|
|
144
|
+
|
|
126
145
|
if event_type:
|
|
127
146
|
self.event = EventNameFilter(name=event_type)
|
|
128
147
|
|
|
@@ -159,6 +178,7 @@ class InfrahubEventFilter(EventFilter):
|
|
|
159
178
|
parent__ids: list[str] | None = None,
|
|
160
179
|
primary_node__ids: list[str] | None = None,
|
|
161
180
|
event_type: list[str] | None = None,
|
|
181
|
+
event_type_filter: dict[str, Any] | None = None,
|
|
162
182
|
branches: list[str] | None = None,
|
|
163
183
|
level: int | None = None,
|
|
164
184
|
has_children: bool | None = None,
|
|
@@ -167,10 +187,10 @@ class InfrahubEventFilter(EventFilter):
|
|
|
167
187
|
) -> InfrahubEventFilter:
|
|
168
188
|
occurred_filter = {}
|
|
169
189
|
if since:
|
|
170
|
-
occurred_filter["since"] = Timestamp(since.isoformat()).
|
|
190
|
+
occurred_filter["since"] = Timestamp(since.isoformat()).to_datetime()
|
|
171
191
|
|
|
172
192
|
if until:
|
|
173
|
-
occurred_filter["until"] = Timestamp(until.isoformat()).
|
|
193
|
+
occurred_filter["until"] = Timestamp(until.isoformat()).to_datetime()
|
|
174
194
|
|
|
175
195
|
if occurred_filter:
|
|
176
196
|
filters = cls(occurred=EventOccurredFilter(**occurred_filter))
|
|
@@ -179,7 +199,7 @@ class InfrahubEventFilter(EventFilter):
|
|
|
179
199
|
|
|
180
200
|
filters.add_event_filter(level=level, has_children=has_children)
|
|
181
201
|
filters.add_event_id_filter(ids=ids)
|
|
182
|
-
filters.add_event_type_filter(event_type=event_type)
|
|
202
|
+
filters.add_event_type_filter(event_type=event_type, event_type_filter=event_type_filter)
|
|
183
203
|
filters.add_branch_filter(branches=branches)
|
|
184
204
|
filters.add_account_filter(account__ids=account__ids)
|
|
185
205
|
filters.add_parent_filter(parent__ids=parent__ids)
|
infrahub/tasks/artifact.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
from typing import Union
|
|
2
|
-
|
|
3
1
|
from infrahub_sdk.node import InfrahubNode
|
|
4
2
|
from prefect import task
|
|
5
3
|
from prefect.cache_policies import NONE
|
|
@@ -13,8 +11,10 @@ from infrahub.services import InfrahubServices
|
|
|
13
11
|
|
|
14
12
|
@task(name="define-artifact", task_run_name="Define Artifact", cache_policy=NONE) # type: ignore[arg-type]
|
|
15
13
|
async def define_artifact(
|
|
16
|
-
model:
|
|
17
|
-
) -> InfrahubNode:
|
|
14
|
+
model: CheckArtifactCreate | RequestArtifactGenerate, service: InfrahubServices
|
|
15
|
+
) -> tuple[InfrahubNode, bool]:
|
|
16
|
+
"""Return an artifact together with a flag to indicate if the artifact is created now or already existed."""
|
|
17
|
+
created = False
|
|
18
18
|
if model.artifact_id:
|
|
19
19
|
artifact = await service.client.get(kind=InfrahubKind.ARTIFACT, id=model.artifact_id, branch=model.branch_name)
|
|
20
20
|
else:
|
|
@@ -39,5 +39,6 @@ async def define_artifact(
|
|
|
39
39
|
"content_type": model.content_type,
|
|
40
40
|
},
|
|
41
41
|
)
|
|
42
|
-
await artifact.save()
|
|
43
|
-
|
|
42
|
+
await artifact.save(request_context=model.context.to_request_context())
|
|
43
|
+
created = True
|
|
44
|
+
return artifact, created
|
|
File without changes
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
from neo4j.exceptions import Neo4jError
|
|
2
|
+
from prefect import task
|
|
3
|
+
from prefect.cache_policies import NONE
|
|
4
|
+
|
|
5
|
+
from infrahub.core import utils
|
|
6
|
+
from infrahub.core.graph.schema import GRAPH_SCHEMA
|
|
7
|
+
from infrahub.core.query import QueryType
|
|
8
|
+
from infrahub.database import DatabaseType, InfrahubDatabase
|
|
9
|
+
|
|
10
|
+
from .models import TelemetryDatabaseData, TelemetryDatabaseServerData, TelemetryDatabaseSystemInfoData
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
async def get_server_info(db: InfrahubDatabase) -> list[TelemetryDatabaseServerData]:
|
|
14
|
+
data: list[TelemetryDatabaseServerData] = []
|
|
15
|
+
|
|
16
|
+
try:
|
|
17
|
+
results = await db.execute_query(query="SHOW SERVERS YIELD *", name="get_server_info", type=QueryType.READ)
|
|
18
|
+
except Neo4jError:
|
|
19
|
+
return []
|
|
20
|
+
|
|
21
|
+
for result in results:
|
|
22
|
+
data.append(
|
|
23
|
+
TelemetryDatabaseServerData(
|
|
24
|
+
name=result["name"],
|
|
25
|
+
version=result["version"],
|
|
26
|
+
)
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
return data
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
async def get_system_info(db: InfrahubDatabase) -> TelemetryDatabaseSystemInfoData:
|
|
33
|
+
query = """
|
|
34
|
+
CALL dbms.queryJmx("java.lang:type=OperatingSystem")
|
|
35
|
+
YIELD attributes
|
|
36
|
+
RETURN
|
|
37
|
+
attributes.AvailableProcessors as processor_available,
|
|
38
|
+
attributes.TotalMemorySize as memory_total,
|
|
39
|
+
attributes.FreeMemorySize as memory_available
|
|
40
|
+
"""
|
|
41
|
+
results = await db.execute_query(query=query, name="get_system_info", type=QueryType.READ)
|
|
42
|
+
|
|
43
|
+
return TelemetryDatabaseSystemInfoData(
|
|
44
|
+
memory_total=results[0]["memory_total"]["value"],
|
|
45
|
+
memory_available=results[0]["memory_available"]["value"],
|
|
46
|
+
processor_available=results[0]["processor_available"]["value"],
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@task(name="telemetry-gather-db", task_run_name="Gather Database Information", cache_policy=NONE)
|
|
51
|
+
async def gather_database_information(db: InfrahubDatabase) -> TelemetryDatabaseData:
|
|
52
|
+
async with db.start_session() as dbs:
|
|
53
|
+
server_info = []
|
|
54
|
+
system_info = None
|
|
55
|
+
database_type = db.db_type.value
|
|
56
|
+
|
|
57
|
+
if db.db_type == DatabaseType.NEO4J:
|
|
58
|
+
server_info = await get_server_info(db=dbs)
|
|
59
|
+
system_info = await get_system_info(db=dbs)
|
|
60
|
+
|
|
61
|
+
# server_info is only available on Neo4j Enterprise
|
|
62
|
+
# so if it's not empty, we can assume the database is of type Enterprise
|
|
63
|
+
if len(server_info) == 0:
|
|
64
|
+
database_type = f"{database_type}-community"
|
|
65
|
+
else:
|
|
66
|
+
database_type = f"{database_type}-enterprise"
|
|
67
|
+
|
|
68
|
+
data = TelemetryDatabaseData(
|
|
69
|
+
database_type=database_type,
|
|
70
|
+
relationship_count={
|
|
71
|
+
"total": await utils.count_relationships(db=dbs),
|
|
72
|
+
},
|
|
73
|
+
node_count={
|
|
74
|
+
"total": await utils.count_nodes(db=dbs),
|
|
75
|
+
},
|
|
76
|
+
servers=server_info,
|
|
77
|
+
system_info=system_info,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
for name in GRAPH_SCHEMA["relationships"]:
|
|
81
|
+
data.relationship_count[name] = await utils.count_relationships(db=dbs, label=name)
|
|
82
|
+
|
|
83
|
+
for name in GRAPH_SCHEMA["nodes"]:
|
|
84
|
+
data.node_count[name] = await utils.count_nodes(db=dbs, label=name)
|
|
85
|
+
|
|
86
|
+
return data
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
from pydantic import BaseModel
|
|
2
|
+
|
|
3
|
+
from .constants import InfrahubType
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class TelemetryWorkerData(BaseModel):
|
|
7
|
+
total: int
|
|
8
|
+
active: int
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TelemetryBranchData(BaseModel):
|
|
12
|
+
total: int
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class TelemetrySchemaData(BaseModel):
|
|
16
|
+
node_count: int
|
|
17
|
+
generic_count: int
|
|
18
|
+
last_update: str
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class TelemetryDatabaseServerData(BaseModel):
|
|
22
|
+
name: str
|
|
23
|
+
version: str
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class TelemetryDatabaseSystemInfoData(BaseModel):
|
|
27
|
+
memory_total: int
|
|
28
|
+
memory_available: int
|
|
29
|
+
processor_available: int
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class TelemetryDatabaseData(BaseModel):
|
|
33
|
+
database_type: str
|
|
34
|
+
relationship_count: dict[str, int]
|
|
35
|
+
node_count: dict[str, int]
|
|
36
|
+
servers: list[TelemetryDatabaseServerData]
|
|
37
|
+
system_info: TelemetryDatabaseSystemInfoData | None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class TelemetryWorkPoolData(BaseModel):
|
|
41
|
+
name: str
|
|
42
|
+
type: str
|
|
43
|
+
total_workers: int
|
|
44
|
+
active_workers: int
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class TelemetryPrefectData(BaseModel):
|
|
48
|
+
events: dict[str, int]
|
|
49
|
+
automations: dict[str, int]
|
|
50
|
+
work_pools: list[TelemetryWorkPoolData]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class TelemetryData(BaseModel):
|
|
54
|
+
deployment_id: str | None
|
|
55
|
+
execution_time: float | None
|
|
56
|
+
infrahub_version: str
|
|
57
|
+
infrahub_type: InfrahubType
|
|
58
|
+
python_version: str
|
|
59
|
+
platform: str
|
|
60
|
+
workers: TelemetryWorkerData
|
|
61
|
+
branches: TelemetryBranchData
|
|
62
|
+
features: dict[str, int]
|
|
63
|
+
schema_info: TelemetrySchemaData
|
|
64
|
+
database: TelemetryDatabaseData
|
|
65
|
+
prefect: TelemetryPrefectData
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
from prefect import task
|
|
4
|
+
from prefect.cache_policies import NONE
|
|
5
|
+
from prefect.client.orchestration import PrefectClient, get_client
|
|
6
|
+
from prefect.client.schemas.objects import WorkerStatus
|
|
7
|
+
|
|
8
|
+
from infrahub.events.utils import get_all_events
|
|
9
|
+
from infrahub.trigger.constants import NAME_SEPARATOR
|
|
10
|
+
from infrahub.trigger.models import TriggerType
|
|
11
|
+
|
|
12
|
+
from .models import TelemetryPrefectData, TelemetryWorkPoolData
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@task(name="telemetry-gather-work-pools", task_run_name="Gather Work Pools", cache_policy=NONE)
|
|
16
|
+
async def gather_prefect_work_pools(client: PrefectClient) -> list[TelemetryWorkPoolData]:
|
|
17
|
+
work_pools = await client.read_work_pools()
|
|
18
|
+
data: list[TelemetryWorkPoolData] = []
|
|
19
|
+
|
|
20
|
+
for pool in work_pools:
|
|
21
|
+
workers = await client.read_workers_for_work_pool(work_pool_name=pool.name)
|
|
22
|
+
data.append(
|
|
23
|
+
TelemetryWorkPoolData(
|
|
24
|
+
name=pool.name,
|
|
25
|
+
type=pool.type,
|
|
26
|
+
total_workers=len(workers),
|
|
27
|
+
active_workers=len([item for item in workers if item.status == WorkerStatus.ONLINE]),
|
|
28
|
+
)
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
return data
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@task(name="telemetry-gather-events", task_run_name="Gather Events", cache_policy=NONE)
|
|
35
|
+
async def gather_prefect_events(client: PrefectClient) -> dict[str, Any]:
|
|
36
|
+
infrahub_events = get_all_events()
|
|
37
|
+
events: dict[str, int] = {}
|
|
38
|
+
|
|
39
|
+
async def count_events(event_name: str) -> int:
|
|
40
|
+
payload = {"filter": {"event": {"name": [event_name]}}}
|
|
41
|
+
response = await client._client.post("/events/count-by/event", json=payload)
|
|
42
|
+
response.raise_for_status()
|
|
43
|
+
data = response.json()
|
|
44
|
+
if not isinstance(data, list) or len(data) == 0:
|
|
45
|
+
return 0
|
|
46
|
+
return data[0]["count"]
|
|
47
|
+
|
|
48
|
+
for event in infrahub_events:
|
|
49
|
+
events[event.event_name] = await count_events(event_name=event.event_name)
|
|
50
|
+
|
|
51
|
+
return events
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@task(name="telemetry-gather-automations", task_run_name="Gather Automations", cache_policy=NONE)
|
|
55
|
+
async def gather_prefect_automations(client: PrefectClient) -> dict[str, Any]:
|
|
56
|
+
automations = await client.read_automations()
|
|
57
|
+
|
|
58
|
+
data: dict[str, Any] = {}
|
|
59
|
+
|
|
60
|
+
for trigger_type in TriggerType:
|
|
61
|
+
data[trigger_type.value] = len(
|
|
62
|
+
[item for item in automations if item.name.startswith(f"{trigger_type.value}{NAME_SEPARATOR}")]
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
return data
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@task(name="telemetry-gather-prefect-information", task_run_name="Gather Prefect Information", cache_policy=NONE)
|
|
69
|
+
async def gather_prefect_information() -> TelemetryPrefectData:
|
|
70
|
+
async with get_client(sync_client=False) as client:
|
|
71
|
+
data = TelemetryPrefectData(
|
|
72
|
+
work_pools=await gather_prefect_work_pools(client=client),
|
|
73
|
+
events=await gather_prefect_events(client=client),
|
|
74
|
+
automations=await gather_prefect_automations(client=client),
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
return data
|
|
@@ -12,44 +12,32 @@ from infrahub import __version__, config
|
|
|
12
12
|
from infrahub.core import registry, utils
|
|
13
13
|
from infrahub.core.branch import Branch
|
|
14
14
|
from infrahub.core.constants import InfrahubKind
|
|
15
|
-
from infrahub.core.graph.schema import GRAPH_SCHEMA
|
|
16
15
|
from infrahub.services import InfrahubServices
|
|
17
16
|
|
|
18
|
-
|
|
19
|
-
|
|
17
|
+
from .constants import TELEMETRY_KIND, TELEMETRY_VERSION
|
|
18
|
+
from .database import gather_database_information
|
|
19
|
+
from .models import (
|
|
20
|
+
TelemetryBranchData,
|
|
21
|
+
TelemetryData,
|
|
22
|
+
TelemetrySchemaData,
|
|
23
|
+
TelemetryWorkerData,
|
|
24
|
+
)
|
|
25
|
+
from .task_manager import gather_prefect_information
|
|
26
|
+
from .utils import determine_infrahub_type
|
|
20
27
|
|
|
21
28
|
|
|
22
|
-
@task(name="telemetry-
|
|
23
|
-
async def
|
|
24
|
-
async with service.database.start_session() as db:
|
|
25
|
-
data: dict[str, Any] = {
|
|
26
|
-
"database_type": db.db_type.value,
|
|
27
|
-
"relationship_count": {"total": await utils.count_relationships(db=db)},
|
|
28
|
-
"node_count": {"total": await utils.count_nodes(db=db)},
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
for name in GRAPH_SCHEMA["relationships"]:
|
|
32
|
-
data["relationship_count"][name] = await utils.count_relationships(db=db, label=name)
|
|
33
|
-
|
|
34
|
-
for name in GRAPH_SCHEMA["nodes"]:
|
|
35
|
-
data["node_count"][name] = await utils.count_nodes(db=db, label=name)
|
|
36
|
-
|
|
37
|
-
return data
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
@task(name="telemetry-schema-information", task_run_name="Gather Schema Information", cache_policy=NONE) # type: ignore[arg-type]
|
|
41
|
-
async def gather_schema_information(branch: Branch) -> dict:
|
|
42
|
-
data: dict[str, Any] = {}
|
|
29
|
+
@task(name="telemetry-schema-information", task_run_name="Gather Schema Information", cache_policy=NONE)
|
|
30
|
+
async def gather_schema_information(branch: Branch) -> TelemetrySchemaData:
|
|
43
31
|
main_schema = registry.schema.get_schema_branch(name=branch.name)
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
32
|
+
return TelemetrySchemaData(
|
|
33
|
+
node_count=len(main_schema.node_names),
|
|
34
|
+
generic_count=len(main_schema.generic_names),
|
|
35
|
+
last_update=branch.schema_changed_at or "",
|
|
36
|
+
)
|
|
49
37
|
|
|
50
38
|
|
|
51
|
-
@task(name="telemetry-feature-information", task_run_name="Gather Feature Information", cache_policy=NONE)
|
|
52
|
-
async def gather_feature_information(service: InfrahubServices) -> dict:
|
|
39
|
+
@task(name="telemetry-feature-information", task_run_name="Gather Feature Information", cache_policy=NONE)
|
|
40
|
+
async def gather_feature_information(service: InfrahubServices) -> dict[str, int]:
|
|
53
41
|
async with service.database.start_session() as db:
|
|
54
42
|
data = {}
|
|
55
43
|
features_to_count = [
|
|
@@ -59,7 +47,9 @@ async def gather_feature_information(service: InfrahubServices) -> dict:
|
|
|
59
47
|
InfrahubKind.GENERICGROUP,
|
|
60
48
|
InfrahubKind.PROFILE,
|
|
61
49
|
InfrahubKind.PROPOSEDCHANGE,
|
|
50
|
+
InfrahubKind.OBJECTTEMPLATE,
|
|
62
51
|
InfrahubKind.TRANSFORM,
|
|
52
|
+
InfrahubKind.WEBHOOK,
|
|
63
53
|
]
|
|
64
54
|
for kind in features_to_count:
|
|
65
55
|
data[kind] = await utils.count_nodes(db=db, label=kind)
|
|
@@ -67,37 +57,39 @@ async def gather_feature_information(service: InfrahubServices) -> dict:
|
|
|
67
57
|
return data
|
|
68
58
|
|
|
69
59
|
|
|
70
|
-
@task(name="telemetry-gather-data", task_run_name="Gather Anonynous Data", cache_policy=NONE)
|
|
71
|
-
async def gather_anonymous_telemetry_data(service: InfrahubServices) ->
|
|
60
|
+
@task(name="telemetry-gather-data", task_run_name="Gather Anonynous Data", cache_policy=NONE)
|
|
61
|
+
async def gather_anonymous_telemetry_data(service: InfrahubServices) -> TelemetryData:
|
|
72
62
|
start_time = time.time()
|
|
73
63
|
|
|
74
64
|
default_branch = registry.get_branch_from_registry()
|
|
75
65
|
workers = await service.component.list_workers(branch=default_branch.name, schema_hash=False)
|
|
76
66
|
|
|
77
|
-
data
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
67
|
+
data = TelemetryData(
|
|
68
|
+
deployment_id=registry.id,
|
|
69
|
+
execution_time=None,
|
|
70
|
+
infrahub_version=__version__,
|
|
71
|
+
infrahub_type=determine_infrahub_type(),
|
|
72
|
+
python_version=platform.python_version(),
|
|
73
|
+
platform=platform.machine(),
|
|
74
|
+
workers=TelemetryWorkerData(
|
|
75
|
+
total=len(workers),
|
|
76
|
+
active=len([w for w in workers if w.active]),
|
|
77
|
+
),
|
|
78
|
+
branches=TelemetryBranchData(
|
|
79
|
+
total=len(registry.branch),
|
|
80
|
+
),
|
|
81
|
+
features=await gather_feature_information(service=service),
|
|
82
|
+
schema_info=await gather_schema_information(branch=default_branch),
|
|
83
|
+
database=await gather_database_information(db=service.database),
|
|
84
|
+
prefect=await gather_prefect_information(),
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
data.execution_time = time.time() - start_time
|
|
96
88
|
|
|
97
89
|
return data
|
|
98
90
|
|
|
99
91
|
|
|
100
|
-
@task(name="telemetry-post-data", task_run_name="Upload data", retries=5, cache_policy=NONE)
|
|
92
|
+
@task(name="telemetry-post-data", task_run_name="Upload data", retries=5, cache_policy=NONE)
|
|
101
93
|
async def post_telemetry_data(service: InfrahubServices, url: str, payload: dict[str, Any]) -> None:
|
|
102
94
|
"""Send the telemetry data to the specified URL, using HTTP POST."""
|
|
103
95
|
response = await service.http.post(url=url, json=payload)
|
|
@@ -114,13 +106,14 @@ async def send_telemetry_push(service: InfrahubServices) -> None:
|
|
|
114
106
|
log.info(f"Pushing anonymous telemetry data to {config.SETTINGS.main.telemetry_endpoint}...")
|
|
115
107
|
|
|
116
108
|
data = await gather_anonymous_telemetry_data(service=service)
|
|
117
|
-
|
|
109
|
+
data_dict = data.model_dump(mode="json")
|
|
110
|
+
log.info(f"Anonymous usage telemetry gathered in {data.execution_time} seconds. | {data_dict}")
|
|
118
111
|
|
|
119
112
|
payload = {
|
|
120
113
|
"kind": TELEMETRY_KIND,
|
|
121
114
|
"payload_format": TELEMETRY_VERSION,
|
|
122
|
-
"data":
|
|
123
|
-
"checksum": hashlib.sha256(json.dumps(
|
|
115
|
+
"data": data_dict,
|
|
116
|
+
"checksum": hashlib.sha256(json.dumps(data_dict).encode()).hexdigest(),
|
|
124
117
|
}
|
|
125
118
|
|
|
126
119
|
await post_telemetry_data(service=service, url=config.SETTINGS.main.telemetry_endpoint, payload=payload)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import importlib.metadata
|
|
2
|
+
|
|
3
|
+
from .constants import InfrahubType
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def determine_infrahub_type() -> InfrahubType:
|
|
7
|
+
try:
|
|
8
|
+
importlib.metadata.version("infrahub-enterprise")
|
|
9
|
+
return InfrahubType.ENTERPRISE
|
|
10
|
+
except importlib.metadata.PackageNotFoundError:
|
|
11
|
+
return InfrahubType.COMMUNITY
|