infrahub-server 1.1.7__py3-none-any.whl → 1.2.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- infrahub/api/artifact.py +16 -4
- infrahub/api/dependencies.py +8 -0
- infrahub/api/oauth2.py +0 -1
- infrahub/api/oidc.py +0 -1
- infrahub/api/query.py +18 -7
- infrahub/api/schema.py +32 -6
- infrahub/api/transformation.py +12 -5
- infrahub/{message_bus/messages/check_artifact_create.py → artifacts/models.py} +5 -3
- infrahub/{message_bus/operations/check/artifact.py → artifacts/tasks.py} +26 -25
- infrahub/cli/__init__.py +0 -2
- infrahub/cli/db.py +6 -7
- infrahub/cli/events.py +8 -3
- infrahub/cli/git_agent.py +9 -7
- infrahub/cli/tasks.py +4 -6
- infrahub/computed_attribute/models.py +1 -1
- infrahub/computed_attribute/tasks.py +64 -17
- infrahub/computed_attribute/triggers.py +90 -0
- infrahub/config.py +1 -1
- infrahub/context.py +39 -0
- infrahub/core/account.py +5 -8
- infrahub/core/attribute.py +50 -21
- infrahub/core/branch/models.py +4 -4
- infrahub/core/branch/tasks.py +130 -125
- infrahub/core/changelog/__init__.py +0 -0
- infrahub/core/changelog/diff.py +283 -0
- infrahub/core/changelog/models.py +499 -0
- infrahub/core/constants/__init__.py +43 -2
- infrahub/core/constants/infrahubkind.py +1 -0
- infrahub/core/constants/schema.py +2 -0
- infrahub/core/diff/combiner.py +1 -1
- infrahub/core/diff/enricher/cardinality_one.py +6 -1
- infrahub/core/diff/enricher/hierarchy.py +22 -7
- infrahub/core/diff/enricher/labels.py +6 -1
- infrahub/core/diff/enricher/path_identifier.py +5 -1
- infrahub/core/diff/enricher/summary_counts.py +107 -0
- infrahub/core/diff/merger/merger.py +3 -1
- infrahub/core/diff/model/path.py +34 -11
- infrahub/core/diff/parent_node_adder.py +78 -0
- infrahub/core/diff/payload_builder.py +13 -2
- infrahub/core/diff/query/all_conflicts.py +1 -1
- infrahub/core/diff/query/artifact.py +1 -1
- infrahub/core/diff/query/delete_query.py +1 -1
- infrahub/core/diff/query/diff_get.py +1 -1
- infrahub/core/diff/query/diff_summary.py +1 -1
- infrahub/core/diff/query/field_specifiers.py +1 -1
- infrahub/core/diff/query/field_summary.py +1 -1
- infrahub/core/diff/query/filters.py +2 -2
- infrahub/core/diff/query/get_conflict_query.py +1 -1
- infrahub/core/diff/query/has_conflicts_query.py +1 -1
- infrahub/core/diff/query/merge.py +3 -3
- infrahub/core/diff/query/merge_tracking_id.py +1 -1
- infrahub/core/diff/query/roots_metadata.py +1 -1
- infrahub/core/diff/query/save.py +191 -185
- infrahub/core/diff/query/summary_counts_enricher.py +52 -5
- infrahub/core/diff/query/time_range_query.py +1 -1
- infrahub/core/diff/query/update_conflict_query.py +1 -1
- infrahub/core/diff/repository/deserializer.py +9 -4
- infrahub/core/diff/repository/repository.py +156 -38
- infrahub/core/diff/tasks.py +13 -12
- infrahub/core/enums.py +1 -1
- infrahub/core/graph/__init__.py +1 -1
- infrahub/core/graph/index.py +3 -0
- infrahub/core/integrity/object_conflict/conflict_recorder.py +1 -1
- infrahub/core/ipam/reconciler.py +1 -1
- infrahub/core/ipam/tasks.py +2 -3
- infrahub/core/manager.py +20 -15
- infrahub/core/merge.py +5 -2
- infrahub/core/migrations/graph/__init__.py +4 -0
- infrahub/core/migrations/graph/m001_add_version_to_graph.py +1 -1
- infrahub/core/migrations/graph/m002_attribute_is_default.py +2 -2
- infrahub/core/migrations/graph/m003_relationship_parent_optional.py +2 -2
- infrahub/core/migrations/graph/m004_add_attr_documentation.py +1 -1
- infrahub/core/migrations/graph/m005_add_rel_read_only.py +1 -1
- infrahub/core/migrations/graph/m006_add_rel_on_delete.py +1 -1
- infrahub/core/migrations/graph/m007_add_rel_allow_override.py +1 -1
- infrahub/core/migrations/graph/m008_add_human_friendly_id.py +1 -1
- infrahub/core/migrations/graph/m009_add_generate_profile_attr.py +1 -1
- infrahub/core/migrations/graph/m010_add_generate_profile_attr_generic.py +1 -1
- infrahub/core/migrations/graph/m011_remove_profile_relationship_schema.py +2 -2
- infrahub/core/migrations/graph/m012_convert_account_generic.py +12 -23
- infrahub/core/migrations/graph/m013_convert_git_password_credential.py +7 -11
- infrahub/core/migrations/graph/m014_remove_index_attr_value.py +2 -2
- infrahub/core/migrations/graph/m015_diff_format_update.py +1 -1
- infrahub/core/migrations/graph/m016_diff_delete_bug_fix.py +1 -1
- infrahub/core/migrations/graph/m017_add_core_profile.py +1 -1
- infrahub/core/migrations/graph/m018_uniqueness_nulls.py +2 -2
- infrahub/core/migrations/graph/m019_restore_rels_to_time.py +256 -0
- infrahub/core/migrations/graph/m020_add_generate_template_attr.py +48 -0
- infrahub/core/migrations/query/attribute_add.py +1 -1
- infrahub/core/migrations/query/attribute_rename.py +1 -1
- infrahub/core/migrations/query/delete_element_in_schema.py +1 -1
- infrahub/core/migrations/query/node_duplicate.py +39 -19
- infrahub/core/migrations/query/relationship_duplicate.py +1 -1
- infrahub/core/migrations/query/schema_attribute_update.py +1 -1
- infrahub/core/migrations/schema/node_attribute_remove.py +1 -1
- infrahub/core/migrations/schema/node_remove.py +27 -13
- infrahub/core/migrations/schema/tasks.py +5 -5
- infrahub/core/migrations/shared.py +4 -4
- infrahub/core/models.py +7 -8
- infrahub/core/node/__init__.py +164 -45
- infrahub/core/node/base.py +1 -1
- infrahub/core/node/delete_validator.py +4 -4
- infrahub/core/node/ipam.py +7 -7
- infrahub/core/node/resource_manager/ip_prefix_pool.py +8 -5
- infrahub/core/node/standard.py +3 -5
- infrahub/core/property.py +1 -1
- infrahub/core/protocols.py +6 -0
- infrahub/core/protocols_base.py +4 -2
- infrahub/core/query/__init__.py +2 -5
- infrahub/core/query/attribute.py +9 -9
- infrahub/core/query/branch.py +5 -5
- infrahub/core/query/delete.py +1 -1
- infrahub/core/query/diff.py +4 -4
- infrahub/core/query/ipam.py +4 -4
- infrahub/core/query/node.py +11 -12
- infrahub/core/query/relationship.py +211 -25
- infrahub/core/query/resource_manager.py +10 -10
- infrahub/core/query/standard_node.py +6 -6
- infrahub/core/query/task.py +3 -3
- infrahub/core/query/task_log.py +1 -1
- infrahub/core/query/utils.py +5 -5
- infrahub/core/registry.py +0 -2
- infrahub/core/relationship/constraints/count.py +1 -1
- infrahub/core/relationship/constraints/peer_kind.py +1 -1
- infrahub/core/relationship/model.py +76 -38
- infrahub/core/schema/__init__.py +6 -4
- infrahub/core/schema/attribute_schema.py +8 -0
- infrahub/core/schema/basenode_schema.py +13 -3
- infrahub/core/schema/definitions/core/__init__.py +153 -0
- infrahub/core/schema/definitions/core/account.py +168 -0
- infrahub/core/schema/definitions/core/artifact.py +127 -0
- infrahub/core/schema/definitions/core/builtin.py +21 -0
- infrahub/core/schema/definitions/core/check.py +60 -0
- infrahub/core/schema/definitions/core/generator.py +96 -0
- infrahub/core/schema/definitions/core/graphql_query.py +77 -0
- infrahub/core/schema/definitions/core/group.py +105 -0
- infrahub/core/schema/definitions/core/ipam.py +252 -0
- infrahub/core/schema/definitions/core/lineage.py +17 -0
- infrahub/core/schema/definitions/core/menu.py +46 -0
- infrahub/core/schema/definitions/core/permission.py +161 -0
- infrahub/core/schema/definitions/core/profile.py +29 -0
- infrahub/core/schema/definitions/core/propose_change.py +88 -0
- infrahub/core/schema/definitions/core/propose_change_comment.py +188 -0
- infrahub/core/schema/definitions/core/propose_change_validator.py +326 -0
- infrahub/core/schema/definitions/core/repository.py +280 -0
- infrahub/core/schema/definitions/core/resource_pool.py +180 -0
- infrahub/core/schema/definitions/core/template.py +12 -0
- infrahub/core/schema/definitions/core/transform.py +87 -0
- infrahub/core/schema/definitions/core/webhook.py +108 -0
- infrahub/core/schema/definitions/internal.py +16 -0
- infrahub/core/schema/generated/genericnode_schema.py +5 -0
- infrahub/core/schema/generated/node_schema.py +5 -0
- infrahub/core/schema/generic_schema.py +5 -1
- infrahub/core/schema/manager.py +45 -42
- infrahub/core/schema/node_schema.py +4 -0
- infrahub/core/schema/profile_schema.py +4 -0
- infrahub/core/schema/relationship_schema.py +10 -2
- infrahub/core/schema/schema_branch.py +260 -16
- infrahub/core/schema/template_schema.py +36 -0
- infrahub/core/task/user_task.py +7 -5
- infrahub/core/timestamp.py +3 -3
- infrahub/core/utils.py +2 -2
- infrahub/core/validators/attribute/choices.py +1 -1
- infrahub/core/validators/attribute/enum.py +1 -1
- infrahub/core/validators/attribute/kind.py +1 -1
- infrahub/core/validators/attribute/length.py +1 -1
- infrahub/core/validators/attribute/optional.py +1 -1
- infrahub/core/validators/attribute/regex.py +1 -1
- infrahub/core/validators/attribute/unique.py +1 -1
- infrahub/core/validators/checks_runner.py +37 -0
- infrahub/core/validators/node/generate_profile.py +1 -1
- infrahub/core/validators/node/hierarchy.py +1 -1
- infrahub/core/validators/query.py +1 -1
- infrahub/core/validators/relationship/count.py +1 -1
- infrahub/core/validators/relationship/optional.py +1 -1
- infrahub/core/validators/relationship/peer.py +1 -1
- infrahub/core/validators/tasks.py +8 -6
- infrahub/core/validators/uniqueness/query.py +1 -1
- infrahub/database/__init__.py +3 -2
- infrahub/database/memgraph.py +1 -1
- infrahub/dependencies/builder/diff/combiner.py +1 -1
- infrahub/dependencies/builder/diff/conflicts_enricher.py +1 -1
- infrahub/dependencies/builder/diff/deserializer.py +4 -2
- infrahub/dependencies/builder/diff/enricher/hierarchy.py +3 -1
- infrahub/dependencies/builder/diff/enricher/summary_counts.py +8 -0
- infrahub/dependencies/builder/diff/parent_node_adder.py +8 -0
- infrahub/events/artifact_action.py +76 -0
- infrahub/events/branch_action.py +50 -21
- infrahub/events/group_action.py +117 -0
- infrahub/events/models.py +164 -51
- infrahub/events/node_action.py +70 -8
- infrahub/events/repository_action.py +8 -8
- infrahub/events/schema_action.py +21 -8
- infrahub/exceptions.py +9 -0
- infrahub/generators/models.py +1 -0
- infrahub/generators/tasks.py +34 -15
- infrahub/git/base.py +3 -5
- infrahub/git/constants.py +0 -1
- infrahub/git/integrator.py +60 -36
- infrahub/git/models.py +80 -1
- infrahub/git/repository.py +7 -8
- infrahub/git/tasks.py +432 -112
- infrahub/git_credential/helper.py +2 -3
- infrahub/graphql/analyzer.py +572 -11
- infrahub/graphql/app.py +34 -26
- infrahub/graphql/auth/query_permission_checker/anonymous_checker.py +5 -5
- infrahub/graphql/auth/query_permission_checker/default_branch_checker.py +4 -4
- infrahub/graphql/auth/query_permission_checker/merge_operation_checker.py +4 -4
- infrahub/graphql/auth/query_permission_checker/object_permission_checker.py +28 -35
- infrahub/graphql/auth/query_permission_checker/super_admin_checker.py +5 -5
- infrahub/graphql/context.py +33 -0
- infrahub/graphql/enums.py +1 -1
- infrahub/graphql/initialization.py +5 -1
- infrahub/graphql/loaders/node.py +2 -2
- infrahub/graphql/manager.py +63 -63
- infrahub/graphql/mutations/account.py +20 -13
- infrahub/graphql/mutations/artifact_definition.py +16 -12
- infrahub/graphql/mutations/branch.py +86 -40
- infrahub/graphql/mutations/computed_attribute.py +24 -15
- infrahub/graphql/mutations/diff.py +33 -17
- infrahub/graphql/mutations/diff_conflict.py +14 -8
- infrahub/graphql/mutations/generator.py +83 -0
- infrahub/graphql/mutations/graphql_query.py +19 -11
- infrahub/graphql/mutations/ipam.py +25 -23
- infrahub/graphql/mutations/main.py +233 -45
- infrahub/graphql/mutations/menu.py +10 -10
- infrahub/graphql/mutations/proposed_change.py +36 -28
- infrahub/graphql/mutations/relationship.py +341 -130
- infrahub/graphql/mutations/repository.py +41 -35
- infrahub/graphql/mutations/resource_manager.py +26 -26
- infrahub/graphql/mutations/schema.py +66 -33
- infrahub/graphql/mutations/tasks.py +10 -7
- infrahub/graphql/parser.py +1 -1
- infrahub/graphql/permissions.py +3 -10
- infrahub/graphql/queries/account.py +22 -18
- infrahub/graphql/queries/branch.py +6 -4
- infrahub/graphql/queries/diff/tree.py +63 -52
- infrahub/graphql/queries/event.py +115 -0
- infrahub/graphql/queries/internal.py +3 -3
- infrahub/graphql/queries/ipam.py +23 -18
- infrahub/graphql/queries/relationship.py +11 -10
- infrahub/graphql/queries/resource_manager.py +37 -25
- infrahub/graphql/queries/search.py +9 -8
- infrahub/graphql/queries/status.py +12 -9
- infrahub/graphql/queries/task.py +11 -9
- infrahub/graphql/resolvers/resolver.py +69 -43
- infrahub/graphql/resolvers/single_relationship.py +16 -10
- infrahub/graphql/schema.py +4 -0
- infrahub/graphql/subscription/__init__.py +1 -1
- infrahub/graphql/subscription/events.py +1 -1
- infrahub/graphql/subscription/graphql_query.py +8 -8
- infrahub/graphql/types/branch.py +2 -2
- infrahub/graphql/types/common.py +6 -1
- infrahub/graphql/types/context.py +12 -0
- infrahub/graphql/types/enums.py +2 -0
- infrahub/graphql/types/event.py +158 -0
- infrahub/graphql/types/interface.py +2 -2
- infrahub/graphql/types/node.py +3 -3
- infrahub/graphql/types/permission.py +2 -2
- infrahub/graphql/types/relationship.py +3 -3
- infrahub/graphql/types/standard_node.py +9 -11
- infrahub/graphql/utils.py +28 -182
- infrahub/groups/tasks.py +2 -3
- infrahub/lock.py +21 -21
- infrahub/menu/generator.py +0 -1
- infrahub/menu/menu.py +116 -138
- infrahub/menu/models.py +4 -4
- infrahub/message_bus/__init__.py +11 -13
- infrahub/message_bus/messages/__init__.py +0 -14
- infrahub/message_bus/messages/check_generator_run.py +1 -3
- infrahub/message_bus/messages/event_branch_merge.py +3 -0
- infrahub/message_bus/messages/proposed_change/request_proposedchange_refreshartifacts.py +6 -0
- infrahub/message_bus/messages/request_proposedchange_pipeline.py +2 -0
- infrahub/message_bus/messages/send_echo_request.py +1 -1
- infrahub/message_bus/operations/__init__.py +4 -13
- infrahub/message_bus/operations/check/__init__.py +2 -2
- infrahub/message_bus/operations/check/generator.py +1 -3
- infrahub/message_bus/operations/event/branch.py +7 -3
- infrahub/message_bus/operations/event/schema.py +1 -1
- infrahub/message_bus/operations/event/worker.py +0 -3
- infrahub/message_bus/operations/finalize/validator.py +1 -1
- infrahub/message_bus/operations/git/file.py +2 -2
- infrahub/message_bus/operations/git/repository.py +1 -1
- infrahub/message_bus/operations/requests/__init__.py +0 -4
- infrahub/message_bus/operations/requests/generator_definition.py +2 -4
- infrahub/message_bus/operations/requests/proposed_change.py +37 -20
- infrahub/message_bus/operations/send/echo.py +1 -1
- infrahub/message_bus/types.py +1 -1
- infrahub/permissions/globals.py +15 -0
- infrahub/pools/prefix.py +29 -165
- infrahub/prefect_server/__init__.py +0 -0
- infrahub/prefect_server/app.py +18 -0
- infrahub/prefect_server/database.py +20 -0
- infrahub/prefect_server/events.py +28 -0
- infrahub/prefect_server/models.py +46 -0
- infrahub/proposed_change/models.py +18 -1
- infrahub/proposed_change/tasks.py +195 -53
- infrahub/pytest_plugin.py +4 -4
- infrahub/server.py +13 -12
- infrahub/services/__init__.py +148 -63
- infrahub/services/adapters/cache/__init__.py +11 -11
- infrahub/services/adapters/cache/nats.py +42 -25
- infrahub/services/adapters/cache/redis.py +3 -11
- infrahub/services/adapters/event/__init__.py +10 -18
- infrahub/services/adapters/http/__init__.py +0 -5
- infrahub/services/adapters/http/httpx.py +22 -15
- infrahub/services/adapters/message_bus/__init__.py +25 -8
- infrahub/services/adapters/message_bus/local.py +9 -7
- infrahub/services/adapters/message_bus/nats.py +14 -8
- infrahub/services/adapters/message_bus/rabbitmq.py +23 -10
- infrahub/services/adapters/workflow/__init__.py +11 -8
- infrahub/services/adapters/workflow/local.py +27 -6
- infrahub/services/adapters/workflow/worker.py +23 -7
- infrahub/services/component.py +43 -40
- infrahub/services/protocols.py +7 -7
- infrahub/services/scheduler.py +30 -29
- infrahub/storage.py +2 -4
- infrahub/task_manager/constants.py +1 -1
- infrahub/task_manager/event.py +261 -0
- infrahub/task_manager/models.py +147 -3
- infrahub/task_manager/task.py +1 -1
- infrahub/tasks/artifact.py +19 -18
- infrahub/tasks/registry.py +1 -1
- infrahub/tasks/telemetry.py +13 -14
- infrahub/transformations/tasks.py +3 -5
- infrahub/trigger/__init__.py +0 -0
- infrahub/trigger/catalogue.py +16 -0
- infrahub/trigger/constants.py +9 -0
- infrahub/trigger/models.py +105 -0
- infrahub/trigger/tasks.py +91 -0
- infrahub/types.py +1 -1
- infrahub/utils.py +1 -1
- infrahub/webhook/constants.py +0 -2
- infrahub/webhook/models.py +161 -40
- infrahub/webhook/tasks.py +123 -202
- infrahub/webhook/triggers.py +27 -0
- infrahub/workers/infrahub_async.py +36 -25
- infrahub/workers/utils.py +63 -0
- infrahub/workflows/catalogue.py +71 -52
- infrahub/workflows/initialization.py +14 -8
- infrahub/workflows/models.py +28 -4
- infrahub/workflows/utils.py +1 -1
- infrahub_sdk/batch.py +2 -2
- infrahub_sdk/client.py +8 -0
- infrahub_sdk/config.py +1 -1
- infrahub_sdk/ctl/branch.py +3 -2
- infrahub_sdk/ctl/check.py +4 -4
- infrahub_sdk/ctl/cli_commands.py +16 -11
- infrahub_sdk/ctl/exceptions.py +0 -6
- infrahub_sdk/ctl/exporter.py +1 -1
- infrahub_sdk/ctl/generator.py +5 -5
- infrahub_sdk/ctl/importer.py +3 -2
- infrahub_sdk/ctl/menu.py +1 -1
- infrahub_sdk/ctl/object.py +1 -1
- infrahub_sdk/ctl/repository.py +23 -15
- infrahub_sdk/ctl/schema.py +2 -2
- infrahub_sdk/ctl/utils.py +6 -21
- infrahub_sdk/ctl/validate.py +2 -1
- infrahub_sdk/data.py +1 -1
- infrahub_sdk/exceptions.py +12 -0
- infrahub_sdk/generator.py +3 -0
- infrahub_sdk/node.py +5 -8
- infrahub_sdk/protocols.py +20 -8
- infrahub_sdk/schema/__init__.py +14 -5
- infrahub_sdk/schema/main.py +7 -0
- infrahub_sdk/task/__init__.py +1 -0
- infrahub_sdk/task/constants.py +3 -0
- infrahub_sdk/task/exceptions.py +25 -0
- infrahub_sdk/task/manager.py +545 -0
- infrahub_sdk/task/models.py +74 -0
- infrahub_sdk/testing/docker.py +30 -0
- infrahub_sdk/timestamp.py +134 -33
- infrahub_sdk/transfer/exporter/json.py +1 -1
- infrahub_sdk/utils.py +39 -1
- infrahub_sdk/yaml.py +2 -3
- {infrahub_server-1.1.7.dist-info → infrahub_server-1.2.0b1.dist-info}/METADATA +7 -6
- {infrahub_server-1.1.7.dist-info → infrahub_server-1.2.0b1.dist-info}/RECORD +383 -339
- infrahub_testcontainers/container.py +2 -3
- infrahub_testcontainers/docker-compose.test.yml +2 -2
- infrahub/core/branch/constants.py +0 -2
- infrahub/core/schema/definitions/core.py +0 -2274
- infrahub/graphql/query.py +0 -52
- infrahub/message_bus/messages/check_repository_checkdefinition.py +0 -20
- infrahub/message_bus/messages/check_repository_mergeconflicts.py +0 -16
- infrahub/message_bus/messages/check_repository_usercheck.py +0 -26
- infrahub/message_bus/messages/request_artifactdefinition_check.py +0 -17
- infrahub/message_bus/messages/request_repository_checks.py +0 -12
- infrahub/message_bus/messages/request_repository_userchecks.py +0 -18
- infrahub/message_bus/operations/check/repository.py +0 -293
- infrahub/message_bus/operations/requests/artifact_definition.py +0 -148
- infrahub/message_bus/operations/requests/repository.py +0 -133
- infrahub/schema/constants.py +0 -1
- infrahub/schema/tasks.py +0 -76
- infrahub/services/adapters/database/__init__.py +0 -9
- infrahub_sdk/ctl/_file.py +0 -13
- /infrahub/{schema → artifacts}/__init__.py +0 -0
- {infrahub_server-1.1.7.dist-info → infrahub_server-1.2.0b1.dist-info}/LICENSE.txt +0 -0
- {infrahub_server-1.1.7.dist-info → infrahub_server-1.2.0b1.dist-info}/WHEEL +0 -0
- {infrahub_server-1.1.7.dist-info → infrahub_server-1.2.0b1.dist-info}/entry_points.txt +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
from collections import defaultdict
|
|
4
|
-
from typing import TYPE_CHECKING,
|
|
4
|
+
from typing import TYPE_CHECKING, TypeVar
|
|
5
5
|
|
|
6
6
|
import ujson
|
|
7
7
|
from infrahub_sdk.uuidt import UUIDT
|
|
@@ -10,34 +10,36 @@ from infrahub.dependencies.registry import build_component_registry
|
|
|
10
10
|
from infrahub.message_bus import InfrahubMessage, Meta
|
|
11
11
|
from infrahub.message_bus.messages import ROUTING_KEY_MAP
|
|
12
12
|
from infrahub.message_bus.operations import execute_message
|
|
13
|
-
from infrahub.services import InfrahubServices
|
|
14
13
|
from infrahub.services.adapters.message_bus import InfrahubMessageBus
|
|
15
14
|
|
|
16
15
|
if TYPE_CHECKING:
|
|
17
|
-
from infrahub.database import InfrahubDatabase
|
|
18
16
|
from infrahub.message_bus.types import MessageTTL
|
|
19
17
|
|
|
20
18
|
ResponseClass = TypeVar("ResponseClass")
|
|
21
19
|
|
|
22
20
|
|
|
23
21
|
class BusSimulator(InfrahubMessageBus):
|
|
24
|
-
def __init__(self
|
|
22
|
+
def __init__(self) -> None:
|
|
25
23
|
self.messages: list[InfrahubMessage] = []
|
|
26
24
|
self.messages_per_routing_key: dict[str, list[InfrahubMessage]] = {}
|
|
27
|
-
self.service: InfrahubServices = InfrahubServices(database=database, message_bus=self)
|
|
28
25
|
self.replies: dict[str, list[InfrahubMessage]] = defaultdict(list)
|
|
29
26
|
build_component_registry()
|
|
30
27
|
|
|
31
28
|
async def publish(
|
|
32
|
-
self,
|
|
29
|
+
self,
|
|
30
|
+
message: InfrahubMessage,
|
|
31
|
+
routing_key: str,
|
|
32
|
+
delay: MessageTTL | None = None, # noqa: ARG002
|
|
33
|
+
is_retry: bool = False, # noqa: ARG002
|
|
33
34
|
) -> None:
|
|
34
35
|
self.messages.append(message)
|
|
35
36
|
if routing_key not in self.messages_per_routing_key:
|
|
36
37
|
self.messages_per_routing_key[routing_key] = []
|
|
37
38
|
self.messages_per_routing_key[routing_key].append(message)
|
|
39
|
+
assert self.service is not None
|
|
38
40
|
await execute_message(routing_key=routing_key, message_body=message.body, service=self.service)
|
|
39
41
|
|
|
40
|
-
async def reply(self, message: InfrahubMessage, routing_key: str) -> None:
|
|
42
|
+
async def reply(self, message: InfrahubMessage, routing_key: str) -> None: # noqa: ARG002
|
|
41
43
|
correlation_id = message.meta.correlation_id or "default"
|
|
42
44
|
self.replies[correlation_id].append(message)
|
|
43
45
|
|
|
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import ssl
|
|
5
|
-
from typing import TYPE_CHECKING, Awaitable, Callable, MutableMapping,
|
|
5
|
+
from typing import TYPE_CHECKING, Awaitable, Callable, MutableMapping, TypeVar
|
|
6
6
|
|
|
7
7
|
import nats
|
|
8
8
|
import ujson
|
|
@@ -33,7 +33,7 @@ async def _add_request_id(message: InfrahubMessage) -> None:
|
|
|
33
33
|
|
|
34
34
|
|
|
35
35
|
class NATSMessageBus(InfrahubMessageBus):
|
|
36
|
-
def __init__(self, settings:
|
|
36
|
+
def __init__(self, component_type: ComponentType, settings: BrokerSettings | None = None) -> None:
|
|
37
37
|
self.settings = settings or config.SETTINGS.broker
|
|
38
38
|
|
|
39
39
|
self.service: InfrahubServices
|
|
@@ -46,9 +46,15 @@ class NATSMessageBus(InfrahubMessageBus):
|
|
|
46
46
|
self.loop = asyncio.get_running_loop()
|
|
47
47
|
self.futures: MutableMapping[str, asyncio.Future] = {}
|
|
48
48
|
|
|
49
|
-
|
|
50
|
-
self.service = service
|
|
49
|
+
self.component_type: ComponentType = component_type
|
|
51
50
|
|
|
51
|
+
@classmethod
|
|
52
|
+
async def new(cls, component_type: ComponentType, settings: BrokerSettings | None = None) -> NATSMessageBus:
|
|
53
|
+
message_bus = cls(component_type=component_type, settings=settings)
|
|
54
|
+
await message_bus._initialize()
|
|
55
|
+
return message_bus
|
|
56
|
+
|
|
57
|
+
async def _initialize(self) -> None:
|
|
52
58
|
tls_context = None
|
|
53
59
|
if self.settings.tls_enabled:
|
|
54
60
|
tls_context = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH)
|
|
@@ -67,9 +73,9 @@ class NATSMessageBus(InfrahubMessageBus):
|
|
|
67
73
|
|
|
68
74
|
self.jetstream = self.connection.jetstream()
|
|
69
75
|
|
|
70
|
-
if self.
|
|
76
|
+
if self.component_type == ComponentType.API_SERVER:
|
|
71
77
|
await self._initialize_api_server()
|
|
72
|
-
elif self.
|
|
78
|
+
elif self.component_type == ComponentType.GIT_AGENT:
|
|
73
79
|
await self._initialize_git_worker()
|
|
74
80
|
|
|
75
81
|
async def shutdown(self) -> None:
|
|
@@ -207,7 +213,7 @@ class NATSMessageBus(InfrahubMessageBus):
|
|
|
207
213
|
await self.publish(message, routing_key)
|
|
208
214
|
|
|
209
215
|
async def publish(
|
|
210
|
-
self, message: InfrahubMessage, routing_key: str, delay:
|
|
216
|
+
self, message: InfrahubMessage, routing_key: str, delay: MessageTTL | None = None, is_retry: bool = False
|
|
211
217
|
) -> None:
|
|
212
218
|
with trace.get_tracer(__name__).start_as_current_span("publish_message") as span:
|
|
213
219
|
span.set_attribute("routing_key", routing_key)
|
|
@@ -282,7 +288,7 @@ class NATSMessageBus(InfrahubMessageBus):
|
|
|
282
288
|
request_id=request_id, correlation_id=correlation_id, reply_to=self.callback_queue.config.name
|
|
283
289
|
)
|
|
284
290
|
|
|
285
|
-
await self.service.send(message=message)
|
|
291
|
+
await self.service.message_bus.send(message=message)
|
|
286
292
|
|
|
287
293
|
response: nats.aio.msg.Msg = await future
|
|
288
294
|
data = ujson.loads(response.data)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
|
-
from typing import TYPE_CHECKING, Awaitable, Callable, MutableMapping,
|
|
4
|
+
from typing import TYPE_CHECKING, Awaitable, Callable, MutableMapping, TypeVar
|
|
5
5
|
|
|
6
6
|
import aio_pika
|
|
7
7
|
import opentelemetry.instrumentation.aio_pika.span_builder
|
|
@@ -65,12 +65,14 @@ async def _add_request_id(message: InfrahubMessage) -> None:
|
|
|
65
65
|
|
|
66
66
|
|
|
67
67
|
class RabbitMQMessageBus(InfrahubMessageBus):
|
|
68
|
-
def __init__(
|
|
68
|
+
def __init__(
|
|
69
|
+
self, component_type: ComponentType = ComponentType.NONE, settings: BrokerSettings | None = None
|
|
70
|
+
) -> None:
|
|
69
71
|
self.settings = settings or config.SETTINGS.broker
|
|
70
72
|
self.channel: AbstractChannel
|
|
71
73
|
self.exchange: AbstractExchange
|
|
72
74
|
self.delayed_exchange: AbstractExchange
|
|
73
|
-
self.service: InfrahubServices
|
|
75
|
+
self.service: InfrahubServices # Needed because we inject `service` while sending messages.
|
|
74
76
|
self.connection: AbstractRobustConnection
|
|
75
77
|
self.callback_queue: AbstractQueue
|
|
76
78
|
self.events_queue: AbstractQueue
|
|
@@ -80,10 +82,17 @@ class RabbitMQMessageBus(InfrahubMessageBus):
|
|
|
80
82
|
self.loop = asyncio.get_running_loop()
|
|
81
83
|
self.futures: MutableMapping[str, asyncio.Future] = {}
|
|
82
84
|
|
|
83
|
-
|
|
85
|
+
self.component_type: ComponentType = component_type
|
|
86
|
+
|
|
87
|
+
@classmethod
|
|
88
|
+
async def new(cls, component_type: ComponentType, settings: BrokerSettings | None = None) -> RabbitMQMessageBus:
|
|
89
|
+
message_bus = cls(component_type=component_type, settings=settings)
|
|
90
|
+
await message_bus._initialize()
|
|
91
|
+
return message_bus
|
|
92
|
+
|
|
93
|
+
async def _initialize(self) -> None:
|
|
84
94
|
patch_spanbuilder_set_channel()
|
|
85
95
|
|
|
86
|
-
self.service = service
|
|
87
96
|
self.connection = await aio_pika.connect_robust(
|
|
88
97
|
host=self.settings.address,
|
|
89
98
|
login=self.settings.username,
|
|
@@ -103,9 +112,9 @@ class RabbitMQMessageBus(InfrahubMessageBus):
|
|
|
103
112
|
async def _initialize_connection(self) -> None:
|
|
104
113
|
self.channel = await self.connection.channel()
|
|
105
114
|
|
|
106
|
-
if self.
|
|
115
|
+
if self.component_type == ComponentType.API_SERVER:
|
|
107
116
|
await self._initialize_api_server()
|
|
108
|
-
elif self.
|
|
117
|
+
elif self.component_type == ComponentType.GIT_AGENT:
|
|
109
118
|
await self._initialize_git_worker()
|
|
110
119
|
|
|
111
120
|
async def shutdown(self) -> None:
|
|
@@ -135,7 +144,7 @@ class RabbitMQMessageBus(InfrahubMessageBus):
|
|
|
135
144
|
|
|
136
145
|
async def on_reconnect(
|
|
137
146
|
self,
|
|
138
|
-
weak: bool = False, #
|
|
147
|
+
weak: bool = False, # noqa: ARG002
|
|
139
148
|
) -> None:
|
|
140
149
|
self.service.log.info("Reconnected to RabbitMQ, reinitializing connection")
|
|
141
150
|
await self._initialize_connection()
|
|
@@ -209,7 +218,11 @@ class RabbitMQMessageBus(InfrahubMessageBus):
|
|
|
209
218
|
await queue.consume(callback=self.on_message, no_ack=False)
|
|
210
219
|
|
|
211
220
|
async def publish(
|
|
212
|
-
self,
|
|
221
|
+
self,
|
|
222
|
+
message: InfrahubMessage,
|
|
223
|
+
routing_key: str,
|
|
224
|
+
delay: MessageTTL | None = None,
|
|
225
|
+
is_retry: bool = False, # noqa: ARG002
|
|
213
226
|
) -> None:
|
|
214
227
|
for enricher in self.message_enrichers:
|
|
215
228
|
await enricher(message)
|
|
@@ -233,7 +246,7 @@ class RabbitMQMessageBus(InfrahubMessageBus):
|
|
|
233
246
|
request_id = log_data.get("request_id", "")
|
|
234
247
|
message.meta = Meta(request_id=request_id, correlation_id=correlation_id, reply_to=self.callback_queue.name)
|
|
235
248
|
|
|
236
|
-
await self.service.send(message=message)
|
|
249
|
+
await self.service.message_bus.send(message=message)
|
|
237
250
|
|
|
238
251
|
response: AbstractIncomingMessage = await future
|
|
239
252
|
data = ujson.loads(response.body)
|
|
@@ -1,26 +1,24 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
3
4
|
from typing import TYPE_CHECKING, Any, Callable, ParamSpec, TypeVar, overload
|
|
4
5
|
|
|
5
6
|
if TYPE_CHECKING:
|
|
6
|
-
from infrahub.
|
|
7
|
+
from infrahub.context import InfrahubContext
|
|
7
8
|
from infrahub.workflows.models import WorkflowDefinition, WorkflowInfo
|
|
8
|
-
|
|
9
9
|
Return = TypeVar("Return")
|
|
10
10
|
Params = ParamSpec("Params")
|
|
11
11
|
|
|
12
12
|
FuncType = Callable[Params, Return]
|
|
13
13
|
|
|
14
14
|
|
|
15
|
-
class InfrahubWorkflow:
|
|
16
|
-
async def initialize(self, service: InfrahubServices) -> None:
|
|
17
|
-
"""Initialize the Workflow engine"""
|
|
18
|
-
|
|
15
|
+
class InfrahubWorkflow(ABC):
|
|
19
16
|
@overload
|
|
20
17
|
async def execute_workflow(
|
|
21
18
|
self,
|
|
22
19
|
workflow: WorkflowDefinition,
|
|
23
20
|
expected_return: type[Return],
|
|
21
|
+
context: InfrahubContext | None = ...,
|
|
24
22
|
parameters: dict[str, Any] | None = ...,
|
|
25
23
|
tags: list[str] | None = ...,
|
|
26
24
|
) -> Return: ...
|
|
@@ -30,23 +28,28 @@ class InfrahubWorkflow:
|
|
|
30
28
|
self,
|
|
31
29
|
workflow: WorkflowDefinition,
|
|
32
30
|
expected_return: None = ...,
|
|
31
|
+
context: InfrahubContext | None = ...,
|
|
33
32
|
parameters: dict[str, Any] | None = ...,
|
|
34
33
|
tags: list[str] | None = ...,
|
|
35
34
|
) -> Any: ...
|
|
36
35
|
|
|
36
|
+
@abstractmethod
|
|
37
37
|
async def execute_workflow(
|
|
38
38
|
self,
|
|
39
39
|
workflow: WorkflowDefinition,
|
|
40
40
|
expected_return: type[Return] | None = None,
|
|
41
|
+
context: InfrahubContext | None = None,
|
|
41
42
|
parameters: dict[str, Any] | None = None,
|
|
42
43
|
tags: list[str] | None = None,
|
|
43
44
|
) -> Any:
|
|
44
|
-
raise NotImplementedError(
|
|
45
|
+
raise NotImplementedError()
|
|
45
46
|
|
|
47
|
+
@abstractmethod
|
|
46
48
|
async def submit_workflow(
|
|
47
49
|
self,
|
|
48
50
|
workflow: WorkflowDefinition,
|
|
51
|
+
context: InfrahubContext | None = None,
|
|
49
52
|
parameters: dict[str, Any] | None = None,
|
|
50
53
|
tags: list[str] | None = None,
|
|
51
54
|
) -> WorkflowInfo:
|
|
52
|
-
raise NotImplementedError(
|
|
55
|
+
raise NotImplementedError()
|
|
@@ -1,27 +1,48 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import uuid
|
|
2
4
|
from typing import Any
|
|
3
5
|
|
|
6
|
+
from typing_extensions import TYPE_CHECKING
|
|
7
|
+
|
|
8
|
+
from infrahub.workers.utils import inject_context_parameter, inject_service_parameter
|
|
4
9
|
from infrahub.workflows.models import WorkflowDefinition, WorkflowInfo
|
|
5
10
|
|
|
6
11
|
from . import InfrahubWorkflow, Return
|
|
7
12
|
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
from infrahub.context import InfrahubContext
|
|
15
|
+
from infrahub.services import InfrahubServices
|
|
16
|
+
|
|
8
17
|
|
|
9
18
|
class WorkflowLocalExecution(InfrahubWorkflow):
|
|
19
|
+
service: InfrahubServices | None = None # needed for local injections
|
|
20
|
+
|
|
10
21
|
async def execute_workflow(
|
|
11
22
|
self,
|
|
12
23
|
workflow: WorkflowDefinition,
|
|
13
|
-
expected_return: type[Return] | None = None,
|
|
24
|
+
expected_return: type[Return] | None = None, # noqa: ARG002
|
|
25
|
+
context: InfrahubContext | None = None,
|
|
14
26
|
parameters: dict[str, Any] | None = None,
|
|
15
|
-
tags: list[str] | None = None,
|
|
27
|
+
tags: list[str] | None = None, # noqa: ARG002
|
|
16
28
|
) -> Any:
|
|
17
|
-
|
|
18
|
-
|
|
29
|
+
if self.service is None:
|
|
30
|
+
raise ValueError("WorkflowLocalExecution.service is not initialized")
|
|
31
|
+
|
|
32
|
+
flow_func = workflow.load_function()
|
|
33
|
+
parameters = dict(parameters) if parameters is not None else {} # avoid mutating input parameters
|
|
34
|
+
inject_service_parameter(func=flow_func, parameters=parameters, service=self.service)
|
|
35
|
+
inject_context_parameter(func=flow_func, parameters=parameters, context=context)
|
|
36
|
+
|
|
37
|
+
parameters = flow_func.validate_parameters(parameters=parameters)
|
|
38
|
+
return await flow_func(**parameters)
|
|
19
39
|
|
|
20
40
|
async def submit_workflow(
|
|
21
41
|
self,
|
|
22
42
|
workflow: WorkflowDefinition,
|
|
43
|
+
context: InfrahubContext | None = None,
|
|
23
44
|
parameters: dict[str, Any] | None = None,
|
|
24
|
-
tags: list[str] | None = None,
|
|
45
|
+
tags: list[str] | None = None, # noqa: ARG002
|
|
25
46
|
) -> WorkflowInfo:
|
|
26
|
-
await self.execute_workflow(workflow=workflow, parameters=parameters)
|
|
47
|
+
await self.execute_workflow(workflow=workflow, context=context, parameters=parameters)
|
|
27
48
|
return WorkflowInfo(id=uuid.uuid4())
|
|
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, overload
|
|
|
5
5
|
from prefect.client.schemas.objects import StateType
|
|
6
6
|
from prefect.deployments import run_deployment
|
|
7
7
|
|
|
8
|
+
from infrahub.workers.utils import inject_context_parameter
|
|
8
9
|
from infrahub.workflows.initialization import setup_task_manager
|
|
9
10
|
from infrahub.workflows.models import WorkflowInfo
|
|
10
11
|
|
|
@@ -13,15 +14,14 @@ from . import InfrahubWorkflow, Return
|
|
|
13
14
|
if TYPE_CHECKING:
|
|
14
15
|
from prefect.client.schemas.objects import FlowRun
|
|
15
16
|
|
|
16
|
-
from infrahub.
|
|
17
|
+
from infrahub.context import InfrahubContext
|
|
17
18
|
from infrahub.workflows.models import WorkflowDefinition
|
|
18
19
|
|
|
19
20
|
|
|
20
21
|
class WorkflowWorkerExecution(InfrahubWorkflow):
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
if await service.component.is_primary_api():
|
|
22
|
+
@staticmethod
|
|
23
|
+
async def initialize(component_is_primary_server: bool) -> None:
|
|
24
|
+
if component_is_primary_server:
|
|
25
25
|
await setup_task_manager()
|
|
26
26
|
|
|
27
27
|
@overload
|
|
@@ -29,6 +29,7 @@ class WorkflowWorkerExecution(InfrahubWorkflow):
|
|
|
29
29
|
self,
|
|
30
30
|
workflow: WorkflowDefinition,
|
|
31
31
|
expected_return: type[Return],
|
|
32
|
+
context: InfrahubContext | None = None,
|
|
32
33
|
parameters: dict[str, Any] | None = ...,
|
|
33
34
|
tags: list[str] | None = ...,
|
|
34
35
|
) -> Return: ...
|
|
@@ -38,17 +39,24 @@ class WorkflowWorkerExecution(InfrahubWorkflow):
|
|
|
38
39
|
self,
|
|
39
40
|
workflow: WorkflowDefinition,
|
|
40
41
|
expected_return: None = ...,
|
|
42
|
+
context: InfrahubContext | None = ...,
|
|
41
43
|
parameters: dict[str, Any] | None = ...,
|
|
42
44
|
tags: list[str] | None = ...,
|
|
43
45
|
) -> Any: ...
|
|
44
46
|
|
|
47
|
+
# TODO Make expected_return mandatory and remove above overloads.
|
|
45
48
|
async def execute_workflow(
|
|
46
49
|
self,
|
|
47
50
|
workflow: WorkflowDefinition,
|
|
48
|
-
expected_return: type[Return] | None = None,
|
|
51
|
+
expected_return: type[Return] | None = None, # noqa: ARG002
|
|
52
|
+
context: InfrahubContext | None = None,
|
|
49
53
|
parameters: dict[str, Any] | None = None,
|
|
50
54
|
tags: list[str] | None = None,
|
|
51
55
|
) -> Any:
|
|
56
|
+
flow_func = workflow.load_function()
|
|
57
|
+
parameters = dict(parameters) if parameters is not None else {}
|
|
58
|
+
inject_context_parameter(func=flow_func, parameters=parameters, context=context)
|
|
59
|
+
|
|
52
60
|
response: FlowRun = await run_deployment(
|
|
53
61
|
name=workflow.full_name, poll_interval=1, parameters=parameters or {}, tags=tags
|
|
54
62
|
) # type: ignore[return-value, misc]
|
|
@@ -61,7 +69,15 @@ class WorkflowWorkerExecution(InfrahubWorkflow):
|
|
|
61
69
|
return await response.state.result(raise_on_failure=True, fetch=True) # type: ignore[call-overload]
|
|
62
70
|
|
|
63
71
|
async def submit_workflow(
|
|
64
|
-
self,
|
|
72
|
+
self,
|
|
73
|
+
workflow: WorkflowDefinition,
|
|
74
|
+
context: InfrahubContext | None = None,
|
|
75
|
+
parameters: dict[str, Any] | None = None,
|
|
76
|
+
tags: list[str] | None = None,
|
|
65
77
|
) -> WorkflowInfo:
|
|
78
|
+
flow_func = workflow.load_function()
|
|
79
|
+
parameters = dict(parameters) if parameters is not None else {}
|
|
80
|
+
inject_context_parameter(func=flow_func, parameters=parameters, context=context)
|
|
81
|
+
|
|
66
82
|
flow_run = await run_deployment(name=workflow.full_name, timeout=0, parameters=parameters or {}, tags=tags) # type: ignore[return-value, misc]
|
|
67
83
|
return WorkflowInfo.from_flow(flow_run=flow_run)
|
infrahub/services/component.py
CHANGED
|
@@ -1,56 +1,59 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import re
|
|
4
|
-
from typing import TYPE_CHECKING, Any
|
|
4
|
+
from typing import TYPE_CHECKING, Any
|
|
5
|
+
|
|
6
|
+
from attr import dataclass
|
|
5
7
|
|
|
6
8
|
from infrahub.components import ComponentType
|
|
7
9
|
from infrahub.core.constants import GLOBAL_BRANCH_NAME
|
|
8
10
|
from infrahub.core.registry import registry
|
|
9
11
|
from infrahub.core.timestamp import Timestamp
|
|
10
|
-
from infrahub.
|
|
12
|
+
from infrahub.log import get_logger
|
|
11
13
|
from infrahub.message_bus import messages
|
|
12
14
|
from infrahub.message_bus.types import KVTTL
|
|
13
15
|
from infrahub.worker import WORKER_IDENTITY
|
|
14
16
|
|
|
15
17
|
if TYPE_CHECKING:
|
|
16
|
-
from infrahub.
|
|
18
|
+
from infrahub.database import InfrahubDatabase
|
|
19
|
+
from infrahub.services import InfrahubCache, InfrahubMessageBus
|
|
17
20
|
|
|
18
21
|
PRIMARY_API_SERVER = "workers:primary:api_server"
|
|
19
22
|
WORKER_MATCH = re.compile(r":worker:([^:]+)")
|
|
20
23
|
|
|
24
|
+
log = get_logger()
|
|
21
25
|
|
|
22
|
-
class InfrahubComponent:
|
|
23
|
-
def __init__(self) -> None:
|
|
24
|
-
self._service: Optional[InfrahubServices] = None
|
|
25
|
-
|
|
26
|
-
@property
|
|
27
|
-
def service(self) -> InfrahubServices:
|
|
28
|
-
if not self._service:
|
|
29
|
-
raise InitializationError("Component has not been initialized")
|
|
30
26
|
|
|
31
|
-
|
|
27
|
+
@dataclass
|
|
28
|
+
class InfrahubComponent:
|
|
29
|
+
cache: InfrahubCache
|
|
30
|
+
db: InfrahubDatabase
|
|
31
|
+
message_bus: InfrahubMessageBus
|
|
32
|
+
component_type: ComponentType
|
|
33
|
+
|
|
34
|
+
@classmethod
|
|
35
|
+
async def new(
|
|
36
|
+
cls, cache: InfrahubCache, db: InfrahubDatabase, message_bus: InfrahubMessageBus, component_type: ComponentType
|
|
37
|
+
) -> InfrahubComponent:
|
|
38
|
+
component = cls(cache=cache, db=db, message_bus=message_bus, component_type=component_type)
|
|
39
|
+
await component.refresh_heartbeat()
|
|
40
|
+
return component
|
|
32
41
|
|
|
33
42
|
@property
|
|
34
43
|
def component_names(self) -> list[str]:
|
|
35
44
|
names = []
|
|
36
|
-
if self.
|
|
45
|
+
if self.component_type == ComponentType.API_SERVER:
|
|
37
46
|
names.append("api_server")
|
|
38
|
-
elif self.
|
|
47
|
+
elif self.component_type == ComponentType.GIT_AGENT:
|
|
39
48
|
names.append("git_agent")
|
|
40
49
|
return names
|
|
41
50
|
|
|
42
|
-
async def
|
|
43
|
-
|
|
44
|
-
self._service = service
|
|
45
|
-
|
|
46
|
-
await self.refresh_heartbeat()
|
|
47
|
-
|
|
48
|
-
async def is_primary_api(self) -> bool:
|
|
49
|
-
primary_identity = await self.service.cache.get(PRIMARY_API_SERVER)
|
|
51
|
+
async def is_primary_gunicorn_worker(self) -> bool:
|
|
52
|
+
primary_identity = await self.cache.get(PRIMARY_API_SERVER)
|
|
50
53
|
return primary_identity == WORKER_IDENTITY
|
|
51
54
|
|
|
52
55
|
async def list_workers(self, branch: str, schema_hash: bool) -> list[WorkerInfo]:
|
|
53
|
-
keys = await self.
|
|
56
|
+
keys = await self.cache.list_keys(filter_pattern="workers:*")
|
|
54
57
|
|
|
55
58
|
workers: dict[str, WorkerInfo] = {}
|
|
56
59
|
for key in keys:
|
|
@@ -64,9 +67,9 @@ class InfrahubComponent:
|
|
|
64
67
|
schema_hash_keys = []
|
|
65
68
|
if schema_hash:
|
|
66
69
|
schema_hash_keys = [key for key in keys if f":schema_hash:branch:{branch}" in key]
|
|
67
|
-
response = await self.
|
|
70
|
+
response = await self.cache.get_values(keys=schema_hash_keys)
|
|
68
71
|
|
|
69
|
-
for key, value in zip(schema_hash_keys, response):
|
|
72
|
+
for key, value in zip(schema_hash_keys, response, strict=False):
|
|
70
73
|
if match := WORKER_MATCH.search(key):
|
|
71
74
|
identity = match.group(1)
|
|
72
75
|
workers[identity].add_value(key=key, value=value)
|
|
@@ -81,15 +84,15 @@ class InfrahubComponent:
|
|
|
81
84
|
hash_value = schema_branch.get_hash()
|
|
82
85
|
|
|
83
86
|
# Use branch name if we cannot find branch id in cache
|
|
84
|
-
branch_id:
|
|
85
|
-
if branch_obj := await registry.get_branch(branch=branch, db=self.
|
|
87
|
+
branch_id: str | None = None
|
|
88
|
+
if branch_obj := await registry.get_branch(branch=branch, db=self.db):
|
|
86
89
|
branch_id = str(branch_obj.uuid)
|
|
87
90
|
|
|
88
91
|
if not branch_id:
|
|
89
92
|
branch_id = branch
|
|
90
93
|
|
|
91
94
|
for component in self.component_names:
|
|
92
|
-
await self.
|
|
95
|
+
await self.cache.set(
|
|
93
96
|
key=f"workers:schema_hash:branch:{branch_id}:{component}:worker:{WORKER_IDENTITY}",
|
|
94
97
|
value=hash_value,
|
|
95
98
|
expires=KVTTL.TWO_HOURS,
|
|
@@ -97,39 +100,39 @@ class InfrahubComponent:
|
|
|
97
100
|
|
|
98
101
|
async def refresh_heartbeat(self) -> None:
|
|
99
102
|
for component in self.component_names:
|
|
100
|
-
await self.
|
|
103
|
+
await self.cache.set(
|
|
101
104
|
key=f"workers:active:{component}:worker:{WORKER_IDENTITY}",
|
|
102
105
|
value=Timestamp().to_string(),
|
|
103
106
|
expires=KVTTL.FIFTEEN,
|
|
104
107
|
)
|
|
105
|
-
if self.
|
|
108
|
+
if self.component_type == ComponentType.API_SERVER:
|
|
106
109
|
await self._set_primary_api_server()
|
|
107
|
-
await self.
|
|
110
|
+
await self.cache.set(
|
|
108
111
|
key=f"workers:worker:{WORKER_IDENTITY}", value=Timestamp().to_string(), expires=KVTTL.TWO_HOURS
|
|
109
112
|
)
|
|
110
113
|
|
|
111
114
|
async def _set_primary_api_server(self) -> None:
|
|
112
|
-
result = await self.
|
|
115
|
+
result = await self.cache.set(
|
|
113
116
|
key=PRIMARY_API_SERVER, value=WORKER_IDENTITY, expires=KVTTL.FIFTEEN, not_exists=True
|
|
114
117
|
)
|
|
115
118
|
if result:
|
|
116
|
-
await self.
|
|
119
|
+
await self.message_bus.send(message=messages.EventWorkerNewPrimaryAPI(worker_id=WORKER_IDENTITY))
|
|
117
120
|
else:
|
|
118
|
-
|
|
119
|
-
primary_id = await self.
|
|
121
|
+
log.debug("Primary node already set")
|
|
122
|
+
primary_id = await self.cache.get(key=PRIMARY_API_SERVER)
|
|
120
123
|
if primary_id == WORKER_IDENTITY:
|
|
121
|
-
|
|
122
|
-
await self.
|
|
124
|
+
log.debug("Primary node set but same as ours, refreshing lifetime")
|
|
125
|
+
await self.cache.set(key=PRIMARY_API_SERVER, value=WORKER_IDENTITY, expires=KVTTL.FIFTEEN)
|
|
123
126
|
|
|
124
127
|
|
|
125
128
|
class WorkerInfo:
|
|
126
129
|
def __init__(self, identity: str) -> None:
|
|
127
130
|
self.id = identity
|
|
128
131
|
self.active = False
|
|
129
|
-
self._schema_hash:
|
|
132
|
+
self._schema_hash: str | None = None
|
|
130
133
|
|
|
131
134
|
@property
|
|
132
|
-
def schema_hash(self) ->
|
|
135
|
+
def schema_hash(self) -> str | None:
|
|
133
136
|
"""Return schema hash provided that the worker is active."""
|
|
134
137
|
if self.active:
|
|
135
138
|
return self._schema_hash
|
|
@@ -140,7 +143,7 @@ class WorkerInfo:
|
|
|
140
143
|
if "workers:active:" in key:
|
|
141
144
|
self.active = True
|
|
142
145
|
|
|
143
|
-
def add_value(self, key: str, value:
|
|
146
|
+
def add_value(self, key: str, value: str | None = None) -> None:
|
|
144
147
|
if ":schema_hash:" in key:
|
|
145
148
|
self._schema_hash = value
|
|
146
149
|
|
infrahub/services/protocols.py
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
from typing import Any,
|
|
1
|
+
from typing import Any, Protocol
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
class InfrahubLogger(Protocol):
|
|
5
|
-
def debug(self, event:
|
|
5
|
+
def debug(self, event: str | None = None, *args: Any, **kw: Any) -> Any:
|
|
6
6
|
"""Send a debug event"""
|
|
7
7
|
|
|
8
|
-
def info(self, event:
|
|
8
|
+
def info(self, event: str | None = None, *args: Any, **kw: Any) -> Any:
|
|
9
9
|
"""Send an info event"""
|
|
10
10
|
|
|
11
|
-
def warning(self, event:
|
|
11
|
+
def warning(self, event: str | None = None, *args: Any, **kw: Any) -> Any:
|
|
12
12
|
"""Send a warning event"""
|
|
13
13
|
|
|
14
|
-
def error(self, event:
|
|
14
|
+
def error(self, event: str | None = None, *args: Any, **kw: Any) -> Any:
|
|
15
15
|
"""Send an error event."""
|
|
16
16
|
|
|
17
|
-
def critical(self, event:
|
|
17
|
+
def critical(self, event: str | None = None, *args: Any, **kw: Any) -> Any:
|
|
18
18
|
"""Send a critical event."""
|
|
19
19
|
|
|
20
|
-
def exception(self, event:
|
|
20
|
+
def exception(self, event: str | None = None, *args: Any, **kw: Any) -> Any:
|
|
21
21
|
"""Send an exception event."""
|