qontract-reconcile 0.10.2.dev361__py3-none-any.whl → 0.10.2.dev430__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {qontract_reconcile-0.10.2.dev361.dist-info → qontract_reconcile-0.10.2.dev430.dist-info}/METADATA +13 -12
- {qontract_reconcile-0.10.2.dev361.dist-info → qontract_reconcile-0.10.2.dev430.dist-info}/RECORD +351 -345
- reconcile/acs_rbac.py +2 -2
- reconcile/aus/advanced_upgrade_service.py +18 -12
- reconcile/aus/base.py +134 -32
- reconcile/aus/cluster_version_data.py +15 -5
- reconcile/aus/models.py +3 -1
- reconcile/aus/ocm_addons_upgrade_scheduler_org.py +1 -0
- reconcile/aus/ocm_upgrade_scheduler.py +8 -1
- reconcile/aus/ocm_upgrade_scheduler_org.py +20 -5
- reconcile/aus/version_gates/sts_version_gate_handler.py +54 -1
- reconcile/automated_actions/config/integration.py +16 -4
- reconcile/aws_account_manager/integration.py +8 -8
- reconcile/aws_account_manager/reconciler.py +3 -3
- reconcile/aws_ami_cleanup/integration.py +8 -12
- reconcile/aws_ami_share.py +69 -62
- reconcile/aws_cloudwatch_log_retention/integration.py +155 -126
- reconcile/aws_iam_keys.py +1 -0
- reconcile/aws_saml_idp/integration.py +12 -4
- reconcile/aws_saml_roles/integration.py +30 -23
- reconcile/aws_version_sync/integration.py +6 -12
- reconcile/change_owners/bundle.py +3 -3
- reconcile/change_owners/change_log_tracking.py +3 -2
- reconcile/change_owners/change_owners.py +1 -1
- reconcile/change_owners/diff.py +0 -2
- reconcile/checkpoint.py +11 -3
- reconcile/cli.py +93 -10
- reconcile/dashdotdb_dora.py +5 -12
- reconcile/dashdotdb_slo.py +1 -1
- reconcile/database_access_manager.py +123 -117
- reconcile/dynatrace_token_provider/integration.py +1 -1
- reconcile/endpoints_discovery/integration.py +4 -1
- reconcile/endpoints_discovery/merge_request.py +1 -1
- reconcile/endpoints_discovery/merge_request_manager.py +8 -8
- reconcile/external_resources/factories.py +4 -6
- reconcile/external_resources/integration.py +1 -1
- reconcile/external_resources/manager.py +8 -6
- reconcile/external_resources/meta.py +0 -1
- reconcile/external_resources/metrics.py +1 -1
- reconcile/external_resources/model.py +19 -15
- reconcile/external_resources/reconciler.py +7 -4
- reconcile/external_resources/secrets_sync.py +4 -7
- reconcile/external_resources/state.py +26 -16
- reconcile/fleet_labeler/integration.py +1 -1
- reconcile/gabi_authorized_users.py +5 -2
- reconcile/gcp_image_mirror.py +2 -2
- reconcile/github_org.py +1 -1
- reconcile/github_owners.py +4 -0
- reconcile/gitlab_housekeeping.py +13 -15
- reconcile/gitlab_members.py +6 -12
- reconcile/gitlab_owners.py +15 -11
- reconcile/gitlab_permissions.py +8 -12
- reconcile/glitchtip_project_alerts/integration.py +3 -1
- reconcile/gql_definitions/acs/acs_instances.py +5 -5
- reconcile/gql_definitions/acs/acs_policies.py +5 -5
- reconcile/gql_definitions/acs/acs_rbac.py +5 -5
- reconcile/gql_definitions/advanced_upgrade_service/aus_clusters.py +5 -5
- reconcile/gql_definitions/advanced_upgrade_service/aus_organization.py +5 -5
- reconcile/gql_definitions/app_interface_metrics_exporter/onboarding_status.py +5 -5
- reconcile/gql_definitions/app_sre_tekton_access_revalidation/roles.py +5 -5
- reconcile/gql_definitions/app_sre_tekton_access_revalidation/users.py +5 -5
- reconcile/gql_definitions/automated_actions/instance.py +46 -7
- reconcile/gql_definitions/aws_account_manager/aws_accounts.py +5 -5
- reconcile/gql_definitions/aws_ami_cleanup/aws_accounts.py +15 -5
- reconcile/gql_definitions/aws_cloudwatch_log_retention/aws_accounts.py +27 -66
- reconcile/gql_definitions/aws_saml_idp/aws_accounts.py +15 -5
- reconcile/gql_definitions/aws_saml_roles/aws_accounts.py +15 -5
- reconcile/gql_definitions/aws_saml_roles/roles.py +5 -5
- reconcile/gql_definitions/aws_version_sync/clusters.py +5 -5
- reconcile/gql_definitions/aws_version_sync/namespaces.py +5 -5
- reconcile/gql_definitions/change_owners/queries/change_types.py +5 -5
- reconcile/gql_definitions/change_owners/queries/self_service_roles.py +5 -5
- reconcile/gql_definitions/cluster_auth_rhidp/clusters.py +5 -5
- reconcile/gql_definitions/common/alerting_services_settings.py +5 -5
- reconcile/gql_definitions/common/app_code_component_repos.py +5 -5
- reconcile/gql_definitions/common/app_interface_custom_messages.py +5 -5
- reconcile/gql_definitions/common/app_interface_dms_settings.py +5 -5
- reconcile/gql_definitions/common/app_interface_repo_settings.py +5 -5
- reconcile/gql_definitions/common/app_interface_roles.py +5 -5
- reconcile/gql_definitions/common/app_interface_state_settings.py +5 -5
- reconcile/gql_definitions/common/app_interface_vault_settings.py +5 -5
- reconcile/gql_definitions/common/app_quay_repos_escalation_policies.py +5 -5
- reconcile/gql_definitions/common/apps.py +5 -5
- reconcile/gql_definitions/common/aws_vpc_requests.py +15 -5
- reconcile/gql_definitions/common/aws_vpcs.py +5 -5
- reconcile/gql_definitions/common/clusters.py +5 -5
- reconcile/gql_definitions/common/clusters_minimal.py +5 -5
- reconcile/gql_definitions/common/clusters_with_dms.py +5 -5
- reconcile/gql_definitions/common/clusters_with_peering.py +5 -5
- reconcile/gql_definitions/common/github_orgs.py +5 -5
- reconcile/gql_definitions/common/jira_settings.py +5 -5
- reconcile/gql_definitions/common/jiralert_settings.py +5 -5
- reconcile/gql_definitions/common/ldap_settings.py +5 -5
- reconcile/gql_definitions/common/namespaces.py +5 -5
- reconcile/gql_definitions/common/namespaces_minimal.py +7 -5
- reconcile/gql_definitions/common/ocm_env_telemeter.py +5 -5
- reconcile/gql_definitions/common/ocm_environments.py +5 -5
- reconcile/gql_definitions/common/pagerduty_instances.py +5 -5
- reconcile/gql_definitions/common/pgp_reencryption_settings.py +5 -5
- reconcile/gql_definitions/common/pipeline_providers.py +5 -5
- reconcile/gql_definitions/common/quay_instances.py +5 -5
- reconcile/gql_definitions/common/quay_orgs.py +5 -5
- reconcile/gql_definitions/common/reserved_networks.py +5 -5
- reconcile/gql_definitions/common/rhcs_provider_settings.py +5 -5
- reconcile/gql_definitions/common/saas_files.py +5 -5
- reconcile/gql_definitions/common/saas_target_namespaces.py +5 -5
- reconcile/gql_definitions/common/saasherder_settings.py +5 -5
- reconcile/gql_definitions/common/slack_workspaces.py +5 -5
- reconcile/gql_definitions/common/smtp_client_settings.py +5 -5
- reconcile/gql_definitions/common/state_aws_account.py +5 -5
- reconcile/gql_definitions/common/users.py +5 -5
- reconcile/gql_definitions/common/users_with_paths.py +5 -5
- reconcile/gql_definitions/cost_report/app_names.py +5 -5
- reconcile/gql_definitions/cost_report/cost_namespaces.py +5 -5
- reconcile/gql_definitions/cost_report/settings.py +5 -5
- reconcile/gql_definitions/dashdotdb_slo/slo_documents_query.py +5 -5
- reconcile/gql_definitions/dynatrace_token_provider/dynatrace_bootstrap_tokens.py +5 -5
- reconcile/gql_definitions/dynatrace_token_provider/token_specs.py +5 -5
- reconcile/gql_definitions/email_sender/apps.py +5 -5
- reconcile/gql_definitions/email_sender/emails.py +5 -5
- reconcile/gql_definitions/email_sender/users.py +5 -5
- reconcile/gql_definitions/endpoints_discovery/apps.py +5 -5
- reconcile/gql_definitions/external_resources/aws_accounts.py +5 -5
- reconcile/gql_definitions/external_resources/external_resources_modules.py +5 -5
- reconcile/gql_definitions/external_resources/external_resources_namespaces.py +33 -6
- reconcile/gql_definitions/external_resources/external_resources_settings.py +5 -5
- reconcile/gql_definitions/external_resources/fragments/external_resources_module_overrides.py +5 -5
- reconcile/gql_definitions/fleet_labeler/fleet_labels.py +5 -5
- reconcile/gql_definitions/fragments/aus_organization.py +5 -5
- reconcile/gql_definitions/fragments/aws_account_common.py +7 -5
- reconcile/gql_definitions/fragments/aws_account_managed.py +5 -5
- reconcile/gql_definitions/fragments/aws_account_sso.py +5 -5
- reconcile/gql_definitions/fragments/aws_infra_management_account.py +5 -5
- reconcile/gql_definitions/fragments/aws_organization.py +33 -0
- reconcile/gql_definitions/fragments/aws_vpc.py +5 -5
- reconcile/gql_definitions/fragments/aws_vpc_request.py +7 -5
- reconcile/gql_definitions/fragments/container_image_mirror.py +5 -5
- reconcile/gql_definitions/fragments/deploy_resources.py +5 -5
- reconcile/gql_definitions/fragments/disable.py +5 -5
- reconcile/gql_definitions/fragments/email_service.py +5 -5
- reconcile/gql_definitions/fragments/email_user.py +5 -5
- reconcile/gql_definitions/fragments/jumphost_common_fields.py +5 -5
- reconcile/gql_definitions/fragments/membership_source.py +5 -5
- reconcile/gql_definitions/fragments/minimal_ocm_organization.py +5 -5
- reconcile/gql_definitions/fragments/oc_connection_cluster.py +5 -5
- reconcile/gql_definitions/fragments/ocm_environment.py +5 -5
- reconcile/gql_definitions/fragments/pipeline_provider_retention.py +5 -5
- reconcile/gql_definitions/fragments/prometheus_instance.py +5 -5
- reconcile/gql_definitions/fragments/resource_limits_requirements.py +5 -5
- reconcile/gql_definitions/fragments/resource_requests_requirements.py +5 -5
- reconcile/gql_definitions/fragments/resource_values.py +5 -5
- reconcile/gql_definitions/fragments/saas_slo_document.py +5 -5
- reconcile/gql_definitions/fragments/saas_target_namespace.py +5 -5
- reconcile/gql_definitions/fragments/serviceaccount_token.py +5 -5
- reconcile/gql_definitions/fragments/terraform_state.py +5 -5
- reconcile/gql_definitions/fragments/upgrade_policy.py +5 -5
- reconcile/gql_definitions/fragments/user.py +5 -5
- reconcile/gql_definitions/fragments/vault_secret.py +5 -5
- reconcile/gql_definitions/gcp/gcp_docker_repos.py +5 -5
- reconcile/gql_definitions/gcp/gcp_projects.py +5 -5
- reconcile/gql_definitions/gitlab_members/gitlab_instances.py +5 -5
- reconcile/gql_definitions/gitlab_members/permissions.py +5 -5
- reconcile/gql_definitions/glitchtip/glitchtip_instance.py +5 -5
- reconcile/gql_definitions/glitchtip/glitchtip_project.py +5 -5
- reconcile/gql_definitions/glitchtip_project_alerts/glitchtip_project.py +5 -5
- reconcile/gql_definitions/integrations/integrations.py +5 -5
- reconcile/gql_definitions/introspection.json +724 -129
- reconcile/gql_definitions/jenkins_configs/jenkins_configs.py +5 -5
- reconcile/gql_definitions/jenkins_configs/jenkins_instances.py +5 -5
- reconcile/gql_definitions/jira/jira_servers.py +5 -5
- reconcile/gql_definitions/jira_permissions_validator/jira_boards_for_permissions_validator.py +9 -5
- reconcile/gql_definitions/jumphosts/jumphosts.py +5 -5
- reconcile/gql_definitions/ldap_groups/roles.py +5 -5
- reconcile/gql_definitions/ldap_groups/settings.py +5 -5
- reconcile/gql_definitions/maintenance/maintenances.py +5 -5
- reconcile/gql_definitions/membershipsources/roles.py +5 -5
- reconcile/gql_definitions/ocm_labels/clusters.py +5 -5
- reconcile/gql_definitions/ocm_labels/organizations.py +5 -5
- reconcile/gql_definitions/openshift_cluster_bots/clusters.py +5 -5
- reconcile/gql_definitions/openshift_groups/managed_groups.py +5 -5
- reconcile/gql_definitions/openshift_groups/managed_roles.py +5 -5
- reconcile/gql_definitions/openshift_serviceaccount_tokens/tokens.py +5 -5
- reconcile/gql_definitions/quay_membership/quay_membership.py +5 -5
- reconcile/gql_definitions/rhcs/certs.py +25 -79
- reconcile/gql_definitions/rhcs/openshift_resource_rhcs_cert.py +43 -0
- reconcile/gql_definitions/rhidp/organizations.py +5 -5
- reconcile/gql_definitions/service_dependencies/jenkins_instance_fragment.py +5 -5
- reconcile/gql_definitions/service_dependencies/service_dependencies.py +5 -5
- reconcile/gql_definitions/sharding/aws_accounts.py +5 -5
- reconcile/gql_definitions/sharding/ocm_organization.py +5 -5
- reconcile/gql_definitions/skupper_network/site_controller_template.py +5 -5
- reconcile/gql_definitions/skupper_network/skupper_networks.py +5 -5
- reconcile/gql_definitions/slack_usergroups/clusters.py +5 -5
- reconcile/gql_definitions/slack_usergroups/permissions.py +5 -5
- reconcile/gql_definitions/slack_usergroups/users.py +5 -5
- reconcile/gql_definitions/slo_documents/slo_documents.py +5 -5
- reconcile/gql_definitions/status_board/status_board.py +5 -5
- reconcile/gql_definitions/statuspage/statuspages.py +5 -5
- reconcile/gql_definitions/templating/template_collection.py +5 -5
- reconcile/gql_definitions/templating/templates.py +5 -5
- reconcile/gql_definitions/terraform_cloudflare_dns/app_interface_cloudflare_dns_settings.py +5 -5
- reconcile/gql_definitions/terraform_cloudflare_dns/terraform_cloudflare_zones.py +5 -5
- reconcile/gql_definitions/terraform_cloudflare_resources/terraform_cloudflare_accounts.py +5 -5
- reconcile/gql_definitions/terraform_cloudflare_resources/terraform_cloudflare_resources.py +5 -5
- reconcile/gql_definitions/terraform_cloudflare_users/app_interface_setting_cloudflare_and_vault.py +5 -5
- reconcile/gql_definitions/terraform_cloudflare_users/terraform_cloudflare_roles.py +5 -5
- reconcile/gql_definitions/terraform_init/aws_accounts.py +19 -5
- reconcile/gql_definitions/terraform_repo/terraform_repo.py +5 -5
- reconcile/gql_definitions/terraform_resources/database_access_manager.py +5 -5
- reconcile/gql_definitions/terraform_resources/terraform_resources_namespaces.py +30 -6
- reconcile/gql_definitions/terraform_tgw_attachments/aws_accounts.py +15 -5
- reconcile/gql_definitions/unleash_feature_toggles/feature_toggles.py +5 -5
- reconcile/gql_definitions/vault_instances/vault_instances.py +5 -5
- reconcile/gql_definitions/vault_policies/vault_policies.py +5 -5
- reconcile/gql_definitions/vpc_peerings_validator/vpc_peerings_validator.py +5 -5
- reconcile/gql_definitions/vpc_peerings_validator/vpc_peerings_validator_peered_cluster_fragment.py +5 -5
- reconcile/integrations_manager.py +3 -3
- reconcile/jenkins_worker_fleets.py +10 -8
- reconcile/jira_permissions_validator.py +237 -122
- reconcile/ldap_groups/integration.py +1 -1
- reconcile/ocm/types.py +35 -57
- reconcile/ocm_aws_infrastructure_access.py +1 -1
- reconcile/ocm_clusters.py +4 -4
- reconcile/ocm_labels/integration.py +3 -2
- reconcile/ocm_machine_pools.py +33 -27
- reconcile/openshift_base.py +113 -4
- reconcile/openshift_cluster_bots.py +1 -1
- reconcile/openshift_namespace_labels.py +1 -1
- reconcile/openshift_namespaces.py +97 -101
- reconcile/openshift_resources_base.py +6 -2
- reconcile/openshift_rhcs_certs.py +74 -37
- reconcile/openshift_rolebindings.py +7 -11
- reconcile/openshift_saas_deploy.py +4 -5
- reconcile/openshift_saas_deploy_change_tester.py +9 -7
- reconcile/openshift_saas_deploy_trigger_cleaner.py +3 -5
- reconcile/openshift_serviceaccount_tokens.py +2 -2
- reconcile/openshift_upgrade_watcher.py +4 -4
- reconcile/oum/labelset.py +5 -3
- reconcile/oum/models.py +1 -4
- reconcile/prometheus_rules_tester/integration.py +3 -3
- reconcile/quay_mirror.py +1 -1
- reconcile/queries.py +131 -0
- reconcile/rhidp/common.py +3 -5
- reconcile/rhidp/sso_client/base.py +16 -5
- reconcile/saas_auto_promotions_manager/merge_request_manager/renderer.py +1 -1
- reconcile/saas_auto_promotions_manager/subscriber.py +4 -3
- reconcile/skupper_network/integration.py +2 -2
- reconcile/slack_usergroups.py +35 -14
- reconcile/sql_query.py +1 -0
- reconcile/status_board.py +6 -6
- reconcile/statuspage/atlassian.py +7 -7
- reconcile/statuspage/integrations/maintenances.py +4 -3
- reconcile/statuspage/page.py +4 -9
- reconcile/statuspage/status.py +5 -8
- reconcile/templating/lib/rendering.py +3 -3
- reconcile/templating/renderer.py +2 -2
- reconcile/terraform_aws_route53.py +7 -1
- reconcile/terraform_cloudflare_dns.py +3 -3
- reconcile/terraform_cloudflare_resources.py +5 -5
- reconcile/terraform_cloudflare_users.py +3 -2
- reconcile/terraform_init/integration.py +187 -23
- reconcile/terraform_repo.py +16 -12
- reconcile/terraform_resources.py +6 -6
- reconcile/terraform_tgw_attachments.py +27 -19
- reconcile/terraform_users.py +7 -0
- reconcile/terraform_vpc_peerings.py +14 -3
- reconcile/terraform_vpc_resources/integration.py +10 -1
- reconcile/typed_queries/aws_account_tags.py +41 -0
- reconcile/typed_queries/cost_report/app_names.py +1 -1
- reconcile/typed_queries/cost_report/cost_namespaces.py +2 -2
- reconcile/typed_queries/saas_files.py +11 -11
- reconcile/typed_queries/status_board.py +2 -2
- reconcile/unleash_feature_toggles/integration.py +4 -2
- reconcile/utils/acs/base.py +6 -3
- reconcile/utils/acs/policies.py +2 -2
- reconcile/utils/aws_api.py +51 -20
- reconcile/utils/aws_api_typed/api.py +38 -9
- reconcile/utils/aws_api_typed/cloudformation.py +149 -0
- reconcile/utils/aws_api_typed/logs.py +73 -0
- reconcile/utils/aws_api_typed/organization.py +4 -2
- reconcile/utils/binary.py +7 -12
- reconcile/utils/datetime_util.py +67 -0
- reconcile/utils/deadmanssnitch_api.py +1 -1
- reconcile/utils/differ.py +2 -3
- reconcile/utils/early_exit_cache.py +11 -12
- reconcile/utils/expiration.py +7 -3
- reconcile/utils/filtering.py +1 -1
- reconcile/utils/gitlab_api.py +7 -5
- reconcile/utils/glitchtip/client.py +6 -2
- reconcile/utils/glitchtip/models.py +25 -28
- reconcile/utils/gql.py +4 -7
- reconcile/utils/helpers.py +1 -1
- reconcile/utils/instrumented_wrappers.py +1 -1
- reconcile/utils/internal_groups/client.py +2 -2
- reconcile/utils/internal_groups/models.py +8 -17
- reconcile/utils/jinja2/utils.py +6 -101
- reconcile/utils/jira_client.py +82 -63
- reconcile/utils/jjb_client.py +7 -10
- reconcile/utils/jobcontroller/controller.py +2 -2
- reconcile/utils/jobcontroller/models.py +17 -1
- reconcile/utils/json.py +43 -1
- reconcile/utils/membershipsources/app_interface_resolver.py +4 -2
- reconcile/utils/membershipsources/models.py +16 -23
- reconcile/utils/membershipsources/resolver.py +4 -2
- reconcile/utils/merge_request_manager/merge_request_manager.py +4 -4
- reconcile/utils/merge_request_manager/parser.py +6 -6
- reconcile/utils/metrics.py +5 -5
- reconcile/utils/models.py +304 -82
- reconcile/utils/mr/app_interface_reporter.py +2 -2
- reconcile/utils/mr/notificator.py +3 -3
- reconcile/utils/mr/update_access_report_base.py +3 -4
- reconcile/utils/mr/user_maintenance.py +3 -2
- reconcile/utils/oc.py +246 -201
- reconcile/utils/oc_filters.py +3 -3
- reconcile/utils/ocm/addons.py +0 -1
- reconcile/utils/ocm/base.py +17 -20
- reconcile/utils/ocm/cluster_groups.py +1 -1
- reconcile/utils/ocm/identity_providers.py +2 -2
- reconcile/utils/ocm/labels.py +1 -1
- reconcile/utils/ocm/products.py +8 -8
- reconcile/utils/ocm/search_filters.py +3 -6
- reconcile/utils/ocm/service_log.py +4 -6
- reconcile/utils/ocm/sre_capability_labels.py +20 -13
- reconcile/utils/openshift_resource.py +8 -3
- reconcile/utils/pagerduty_api.py +10 -7
- reconcile/utils/promotion_state.py +6 -11
- reconcile/utils/raw_github_api.py +1 -1
- reconcile/utils/rhcsv2_certs.py +86 -23
- reconcile/utils/rosa/session.py +16 -0
- reconcile/utils/runtime/integration.py +2 -3
- reconcile/utils/runtime/runner.py +2 -2
- reconcile/utils/saasherder/interfaces.py +13 -20
- reconcile/utils/saasherder/models.py +23 -20
- reconcile/utils/saasherder/saasherder.py +50 -27
- reconcile/utils/slack_api.py +2 -2
- reconcile/utils/sloth.py +171 -2
- reconcile/utils/structs.py +1 -1
- reconcile/utils/terraform_client.py +5 -4
- reconcile/utils/terrascript_aws_client.py +134 -74
- reconcile/utils/unleash/server.py +2 -8
- reconcile/utils/vault.py +5 -12
- reconcile/utils/vcs.py +8 -8
- reconcile/vault_replication.py +107 -42
- tools/app_interface_reporter.py +4 -4
- tools/cli_commands/cost_report/cost_management_api.py +3 -3
- tools/cli_commands/cost_report/view.py +7 -6
- tools/cli_commands/erv2.py +1 -1
- tools/qontract_cli.py +28 -17
- tools/template_validation.py +3 -1
- {qontract_reconcile-0.10.2.dev361.dist-info → qontract_reconcile-0.10.2.dev430.dist-info}/WHEEL +0 -0
- {qontract_reconcile-0.10.2.dev361.dist-info → qontract_reconcile-0.10.2.dev430.dist-info}/entry_points.txt +0 -0
|
@@ -22,7 +22,6 @@ from typing import (
|
|
|
22
22
|
TYPE_CHECKING,
|
|
23
23
|
Any,
|
|
24
24
|
Self,
|
|
25
|
-
TypeAlias,
|
|
26
25
|
cast,
|
|
27
26
|
)
|
|
28
27
|
|
|
@@ -152,6 +151,7 @@ from reconcile.github_org import get_default_config
|
|
|
152
151
|
from reconcile.gql_definitions.terraform_resources.terraform_resources_namespaces import (
|
|
153
152
|
NamespaceTerraformResourceLifecycleV1,
|
|
154
153
|
)
|
|
154
|
+
from reconcile.typed_queries.aws_account_tags import get_aws_account_tags
|
|
155
155
|
from reconcile.utils import gql
|
|
156
156
|
from reconcile.utils.aws_api import (
|
|
157
157
|
AmiTag,
|
|
@@ -178,7 +178,10 @@ from reconcile.utils.external_resources import (
|
|
|
178
178
|
from reconcile.utils.git import is_file_in_git_repo
|
|
179
179
|
from reconcile.utils.gitlab_api import GitLabApi
|
|
180
180
|
from reconcile.utils.jenkins_api import JenkinsApi
|
|
181
|
-
from reconcile.utils.jinja2.utils import
|
|
181
|
+
from reconcile.utils.jinja2.utils import (
|
|
182
|
+
process_extracurlyjinja2_template,
|
|
183
|
+
process_jinja2_template,
|
|
184
|
+
)
|
|
182
185
|
from reconcile.utils.json import json_dumps
|
|
183
186
|
from reconcile.utils.password_validator import (
|
|
184
187
|
PasswordPolicy,
|
|
@@ -203,7 +206,7 @@ if TYPE_CHECKING:
|
|
|
203
206
|
from reconcile.utils.ocm import OCMMap
|
|
204
207
|
|
|
205
208
|
|
|
206
|
-
TFResource
|
|
209
|
+
type TFResource = type[
|
|
207
210
|
Resource | Data | Module | Provider | Variable | Output | Locals | Terraform
|
|
208
211
|
]
|
|
209
212
|
|
|
@@ -268,6 +271,7 @@ VARIABLE_KEYS = [
|
|
|
268
271
|
"extra_tags",
|
|
269
272
|
"lifecycle",
|
|
270
273
|
"max_session_duration",
|
|
274
|
+
"secret_format",
|
|
271
275
|
]
|
|
272
276
|
|
|
273
277
|
EMAIL_REGEX = re.compile(r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$")
|
|
@@ -470,10 +474,10 @@ class TerrascriptClient:
|
|
|
470
474
|
integration_prefix: str,
|
|
471
475
|
thread_pool_size: int,
|
|
472
476
|
accounts: Iterable[MutableMapping[str, Any]],
|
|
477
|
+
default_tags: Mapping[str, str] | None,
|
|
473
478
|
settings: Mapping[str, Any] | None = None,
|
|
474
479
|
prefetch_resources_by_schemas: Iterable[str] | None = None,
|
|
475
480
|
secret_reader: SecretReaderBase | None = None,
|
|
476
|
-
default_tags: Mapping[str, str] | None = None,
|
|
477
481
|
) -> None:
|
|
478
482
|
self.integration = integration
|
|
479
483
|
self.integration_prefix = integration_prefix
|
|
@@ -484,16 +488,11 @@ class TerrascriptClient:
|
|
|
484
488
|
else:
|
|
485
489
|
self.secret_reader = SecretReader(settings=settings)
|
|
486
490
|
self.configs: dict[str, dict] = {}
|
|
491
|
+
self.default_tags = default_tags or {"app": "app-sre-infra"}
|
|
487
492
|
self.populate_configs(filtered_accounts)
|
|
488
493
|
self.versions: dict[str, str] = {
|
|
489
494
|
a["name"]: a["providerVersion"] for a in filtered_accounts
|
|
490
495
|
}
|
|
491
|
-
self.default_tags = {
|
|
492
|
-
"tags": default_tags
|
|
493
|
-
or {
|
|
494
|
-
"app": "app-sre-infra",
|
|
495
|
-
}
|
|
496
|
-
}
|
|
497
496
|
tss = {}
|
|
498
497
|
locks = {}
|
|
499
498
|
self.supported_regions = {}
|
|
@@ -510,7 +509,7 @@ class TerrascriptClient:
|
|
|
510
509
|
region=region,
|
|
511
510
|
alias=region,
|
|
512
511
|
skip_region_validation=True,
|
|
513
|
-
default_tags=
|
|
512
|
+
default_tags={"tags": config["tags"]},
|
|
514
513
|
)
|
|
515
514
|
|
|
516
515
|
# Add default region, which will be in resourcesDefaultRegion
|
|
@@ -519,7 +518,7 @@ class TerrascriptClient:
|
|
|
519
518
|
secret_key=config["aws_secret_access_key"],
|
|
520
519
|
region=config["resourcesDefaultRegion"],
|
|
521
520
|
skip_region_validation=True,
|
|
522
|
-
default_tags=
|
|
521
|
+
default_tags={"tags": config["tags"]},
|
|
523
522
|
)
|
|
524
523
|
|
|
525
524
|
ts += Terraform(
|
|
@@ -802,6 +801,9 @@ class TerrascriptClient:
|
|
|
802
801
|
config["supportedDeploymentRegions"] = account["supportedDeploymentRegions"]
|
|
803
802
|
config["resourcesDefaultRegion"] = account["resourcesDefaultRegion"]
|
|
804
803
|
config["terraformState"] = account["terraformState"]
|
|
804
|
+
config["tags"] = dict(self.default_tags) | get_aws_account_tags(
|
|
805
|
+
account.get("organization", None)
|
|
806
|
+
)
|
|
805
807
|
self.configs[account_name] = config
|
|
806
808
|
|
|
807
809
|
def _get_partition(self, account: str) -> str:
|
|
@@ -1056,7 +1058,9 @@ class TerrascriptClient:
|
|
|
1056
1058
|
ignore_changes = (
|
|
1057
1059
|
"all" if "all" in lifecycle.ignore_changes else lifecycle.ignore_changes
|
|
1058
1060
|
)
|
|
1059
|
-
return lifecycle.
|
|
1061
|
+
return lifecycle.model_dump(by_alias=True) | {
|
|
1062
|
+
"ignore_changes": ignore_changes
|
|
1063
|
+
}
|
|
1060
1064
|
return None
|
|
1061
1065
|
|
|
1062
1066
|
def populate_additional_providers(
|
|
@@ -1071,25 +1075,15 @@ class TerrascriptClient:
|
|
|
1071
1075
|
config = self.configs[account_name]
|
|
1072
1076
|
existing_provider_aliases = {p.get("alias") for p in ts["provider"]["aws"]}
|
|
1073
1077
|
if alias not in existing_provider_aliases:
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
)
|
|
1084
|
-
else:
|
|
1085
|
-
ts += provider.aws(
|
|
1086
|
-
access_key=config["aws_access_key_id"],
|
|
1087
|
-
secret_key=config["aws_secret_access_key"],
|
|
1088
|
-
region=region,
|
|
1089
|
-
alias=alias,
|
|
1090
|
-
skip_region_validation=True,
|
|
1091
|
-
default_tags=self.default_tags,
|
|
1092
|
-
)
|
|
1078
|
+
ts += provider.aws(
|
|
1079
|
+
access_key=config["aws_access_key_id"],
|
|
1080
|
+
secret_key=config["aws_secret_access_key"],
|
|
1081
|
+
region=region,
|
|
1082
|
+
alias=alias,
|
|
1083
|
+
skip_region_validation=True,
|
|
1084
|
+
default_tags={"tags": config["tags"]},
|
|
1085
|
+
**{"assume_role": {"role_arn": assume_role}} if assume_role else {},
|
|
1086
|
+
)
|
|
1093
1087
|
|
|
1094
1088
|
def populate_route53(
|
|
1095
1089
|
self, desired_state: Iterable[Mapping[str, Any]], default_ttl: int = 300
|
|
@@ -1432,7 +1426,7 @@ class TerrascriptClient:
|
|
|
1432
1426
|
req_account_name = req_account.name
|
|
1433
1427
|
# Accepter's side of the connection - the cluster's account
|
|
1434
1428
|
acc_account = accepter.account
|
|
1435
|
-
acc_alias = self.get_provider_alias(acc_account.
|
|
1429
|
+
acc_alias = self.get_provider_alias(acc_account.model_dump(by_alias=True))
|
|
1436
1430
|
acc_uid = acc_account.uid
|
|
1437
1431
|
if acc_account.assume_role:
|
|
1438
1432
|
acc_uid = awsh.get_account_uid_from_arn(acc_account.assume_role)
|
|
@@ -2218,6 +2212,43 @@ class TerrascriptClient:
|
|
|
2218
2212
|
letters_and_digits = string.ascii_letters + string.digits
|
|
2219
2213
|
return "".join(random.choice(letters_and_digits) for i in range(string_length))
|
|
2220
2214
|
|
|
2215
|
+
@staticmethod
|
|
2216
|
+
def _build_tf_resource_s3_lifecycle_rules(
|
|
2217
|
+
versioning: bool,
|
|
2218
|
+
common_values: Mapping[str, Any],
|
|
2219
|
+
) -> list[dict]:
|
|
2220
|
+
lifecycle_rules = common_values.get("lifecycle_rules") or []
|
|
2221
|
+
if versioning and not any(
|
|
2222
|
+
"noncurrent_version_expiration" in lr for lr in lifecycle_rules
|
|
2223
|
+
):
|
|
2224
|
+
# Add a default noncurrent object expiration rule
|
|
2225
|
+
# if one isn't already set
|
|
2226
|
+
rule = {
|
|
2227
|
+
"id": "expire_noncurrent_versions",
|
|
2228
|
+
"enabled": True,
|
|
2229
|
+
"noncurrent_version_expiration": {"days": 30},
|
|
2230
|
+
"expiration": {"expired_object_delete_marker": True},
|
|
2231
|
+
"abort_incomplete_multipart_upload_days": 3,
|
|
2232
|
+
}
|
|
2233
|
+
lifecycle_rules.append(rule)
|
|
2234
|
+
|
|
2235
|
+
if storage_class := common_values.get("storage_class"):
|
|
2236
|
+
sc = storage_class.upper()
|
|
2237
|
+
days = "1"
|
|
2238
|
+
if sc.endswith("_IA"):
|
|
2239
|
+
# Infrequent Access storage class has minimum 30 days
|
|
2240
|
+
# before transition
|
|
2241
|
+
days = "30"
|
|
2242
|
+
rule = {
|
|
2243
|
+
"id": sc + "_storage_class",
|
|
2244
|
+
"enabled": True,
|
|
2245
|
+
"transition": {"days": days, "storage_class": sc},
|
|
2246
|
+
"noncurrent_version_transition": {"days": days, "storage_class": sc},
|
|
2247
|
+
}
|
|
2248
|
+
lifecycle_rules.append(rule)
|
|
2249
|
+
|
|
2250
|
+
return lifecycle_rules
|
|
2251
|
+
|
|
2221
2252
|
def populate_tf_resource_s3(self, spec: ExternalResourceSpec) -> aws_s3_bucket:
|
|
2222
2253
|
account = spec.provisioner_name
|
|
2223
2254
|
identifier = spec.identifier
|
|
@@ -2257,47 +2288,11 @@ class TerrascriptClient:
|
|
|
2257
2288
|
request_payer = common_values.get("request_payer")
|
|
2258
2289
|
if request_payer:
|
|
2259
2290
|
values["request_payer"] = request_payer
|
|
2260
|
-
lifecycle_rules
|
|
2261
|
-
|
|
2262
|
-
|
|
2291
|
+
if lifecycle_rules := self._build_tf_resource_s3_lifecycle_rules(
|
|
2292
|
+
versioning=versioning,
|
|
2293
|
+
common_values=common_values,
|
|
2294
|
+
):
|
|
2263
2295
|
values["lifecycle_rule"] = lifecycle_rules
|
|
2264
|
-
if versioning:
|
|
2265
|
-
lrs = values.get("lifecycle_rule", [])
|
|
2266
|
-
expiration_rule = False
|
|
2267
|
-
for lr in lrs:
|
|
2268
|
-
if "noncurrent_version_expiration" in lr:
|
|
2269
|
-
expiration_rule = True
|
|
2270
|
-
break
|
|
2271
|
-
if not expiration_rule:
|
|
2272
|
-
# Add a default noncurrent object expiration rule if
|
|
2273
|
-
# if one isn't already set
|
|
2274
|
-
rule = {
|
|
2275
|
-
"id": "expire_noncurrent_versions",
|
|
2276
|
-
"enabled": "true",
|
|
2277
|
-
"noncurrent_version_expiration": {"days": 30},
|
|
2278
|
-
}
|
|
2279
|
-
if len(lrs) > 0:
|
|
2280
|
-
lrs.append(rule)
|
|
2281
|
-
else:
|
|
2282
|
-
lrs = rule
|
|
2283
|
-
sc = common_values.get("storage_class")
|
|
2284
|
-
if sc:
|
|
2285
|
-
sc = sc.upper()
|
|
2286
|
-
days = "1"
|
|
2287
|
-
if sc.endswith("_IA"):
|
|
2288
|
-
# Infrequent Access storage class has minimum 30 days
|
|
2289
|
-
# before transition
|
|
2290
|
-
days = "30"
|
|
2291
|
-
rule = {
|
|
2292
|
-
"id": sc + "_storage_class",
|
|
2293
|
-
"enabled": "true",
|
|
2294
|
-
"transition": {"days": days, "storage_class": sc},
|
|
2295
|
-
"noncurrent_version_transition": {"days": days, "storage_class": sc},
|
|
2296
|
-
}
|
|
2297
|
-
if values.get("lifecycle_rule"):
|
|
2298
|
-
values["lifecycle_rule"].append(rule)
|
|
2299
|
-
else:
|
|
2300
|
-
values["lifecycle_rule"] = rule
|
|
2301
2296
|
cors_rules = common_values.get("cors_rules")
|
|
2302
2297
|
if cors_rules:
|
|
2303
2298
|
# common_values['cors_rules'] is a list of cors_rules
|
|
@@ -5810,6 +5805,10 @@ class TerrascriptClient:
|
|
|
5810
5805
|
assert secret # make mypy happy
|
|
5811
5806
|
secret_data = self.secret_reader.read_all(secret)
|
|
5812
5807
|
|
|
5808
|
+
secret_format = common_values.get("secret_format")
|
|
5809
|
+
if secret_format is not None:
|
|
5810
|
+
secret_data = self._apply_secret_format(str(secret_format), secret_data)
|
|
5811
|
+
|
|
5813
5812
|
version_values: dict[str, Any] = {
|
|
5814
5813
|
"secret_id": "${" + aws_secret_resource.id + "}",
|
|
5815
5814
|
"secret_string": json_dumps(secret_data),
|
|
@@ -5833,6 +5832,66 @@ class TerrascriptClient:
|
|
|
5833
5832
|
|
|
5834
5833
|
self.add_resources(account, tf_resources)
|
|
5835
5834
|
|
|
5835
|
+
@staticmethod
|
|
5836
|
+
def _unflatten_dotted_keys_dict(flat_dict: dict[str, str]) -> dict[str, Any]:
|
|
5837
|
+
"""Convert a flat dictionary with dotted keys to a nested dictionary.
|
|
5838
|
+
|
|
5839
|
+
Example:
|
|
5840
|
+
{"db.host": "localhost", "db.port": "5432"} ->
|
|
5841
|
+
{"db": {"host": "localhost", "port": "5432"}}
|
|
5842
|
+
|
|
5843
|
+
Raises:
|
|
5844
|
+
ValueError: If there are conflicting keys (e.g., "a.b" and "a.b.c")
|
|
5845
|
+
"""
|
|
5846
|
+
result: dict[str, Any] = {}
|
|
5847
|
+
for key, value in flat_dict.items():
|
|
5848
|
+
parts = key.split(".")
|
|
5849
|
+
current = result
|
|
5850
|
+
for i, part in enumerate(parts[:-1]):
|
|
5851
|
+
if part not in current:
|
|
5852
|
+
current[part] = {}
|
|
5853
|
+
elif not isinstance(current[part], dict):
|
|
5854
|
+
# Conflict: trying to traverse through a non-dict value
|
|
5855
|
+
conflicting_path = ".".join(parts[: i + 1])
|
|
5856
|
+
raise ValueError(
|
|
5857
|
+
f"Conflicting keys detected: '{conflicting_path}' is both a "
|
|
5858
|
+
f"value and a nested path in key '{key}'"
|
|
5859
|
+
)
|
|
5860
|
+
current = current[part]
|
|
5861
|
+
|
|
5862
|
+
# Check if we're trying to set a value where a dict already exists
|
|
5863
|
+
if parts[-1] in current and isinstance(current[parts[-1]], dict):
|
|
5864
|
+
raise ValueError(
|
|
5865
|
+
f"Conflicting keys detected: '{key}' conflicts with nested keys"
|
|
5866
|
+
)
|
|
5867
|
+
|
|
5868
|
+
current[parts[-1]] = value
|
|
5869
|
+
|
|
5870
|
+
return result
|
|
5871
|
+
|
|
5872
|
+
@staticmethod
|
|
5873
|
+
def _apply_secret_format(
|
|
5874
|
+
secret_format: str, secret_data: dict[str, str]
|
|
5875
|
+
) -> dict[str, str]:
|
|
5876
|
+
# Convert flat dict with dotted keys to nested dict for Jinja2
|
|
5877
|
+
nested_secret_data = TerrascriptClient._unflatten_dotted_keys_dict(secret_data)
|
|
5878
|
+
rendered_data = process_jinja2_template(secret_format, nested_secret_data)
|
|
5879
|
+
|
|
5880
|
+
parsed_data = json.loads(rendered_data)
|
|
5881
|
+
|
|
5882
|
+
if not isinstance(parsed_data, dict):
|
|
5883
|
+
raise ValueError("secret_format must be a dictionary")
|
|
5884
|
+
|
|
5885
|
+
# validate secret is a dict[str, str]
|
|
5886
|
+
for k, v in parsed_data.items():
|
|
5887
|
+
if not isinstance(k, str):
|
|
5888
|
+
raise ValueError(f"key '{k}' is not a string")
|
|
5889
|
+
|
|
5890
|
+
if not isinstance(v, str):
|
|
5891
|
+
raise ValueError(f"dictionary value '{v}' under '{k}' is not a string")
|
|
5892
|
+
|
|
5893
|
+
return parsed_data
|
|
5894
|
+
|
|
5836
5895
|
def get_commit_sha(self, repo_info: Mapping) -> str:
|
|
5837
5896
|
url = repo_info["url"]
|
|
5838
5897
|
ref = repo_info["ref"]
|
|
@@ -5851,7 +5910,8 @@ class TerrascriptClient:
|
|
|
5851
5910
|
return commit.sha
|
|
5852
5911
|
case "gitlab":
|
|
5853
5912
|
gitlab = self.init_gitlab()
|
|
5854
|
-
project
|
|
5913
|
+
if not (project := gitlab.get_project(url)):
|
|
5914
|
+
raise ValueError(f"could not find gitlab project for url {url}")
|
|
5855
5915
|
commits = project.commits.list(ref_name=ref, per_page=1, page=1)
|
|
5856
5916
|
return commits[0].id
|
|
5857
5917
|
case _:
|
|
@@ -24,30 +24,24 @@ class Environment(BaseModel):
|
|
|
24
24
|
return self.name == other
|
|
25
25
|
|
|
26
26
|
|
|
27
|
-
class FeatureToggle(BaseModel):
|
|
27
|
+
class FeatureToggle(BaseModel, validate_by_name=True, validate_by_alias=True):
|
|
28
28
|
name: str
|
|
29
29
|
type: FeatureToggleType = FeatureToggleType.release
|
|
30
30
|
description: str | None = None
|
|
31
31
|
impression_data: bool = Field(False, alias="impressionData")
|
|
32
32
|
environments: list[Environment]
|
|
33
33
|
|
|
34
|
-
class Config:
|
|
35
|
-
allow_population_by_field_name = True
|
|
36
|
-
|
|
37
34
|
def __eq__(self, other: object) -> bool:
|
|
38
35
|
if isinstance(other, FeatureToggle):
|
|
39
36
|
return self.name == other.name
|
|
40
37
|
return self.name == other
|
|
41
38
|
|
|
42
39
|
|
|
43
|
-
class Project(BaseModel):
|
|
40
|
+
class Project(BaseModel, validate_by_name=True, validate_by_alias=True):
|
|
44
41
|
pk: str = Field(alias="id")
|
|
45
42
|
name: str
|
|
46
43
|
feature_toggles: list[FeatureToggle] = []
|
|
47
44
|
|
|
48
|
-
class Config:
|
|
49
|
-
allow_population_by_field_name = True
|
|
50
|
-
|
|
51
45
|
|
|
52
46
|
class TokenAuth(BearerTokenAuth):
|
|
53
47
|
def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest:
|
reconcile/utils/vault.py
CHANGED
|
@@ -6,7 +6,7 @@ import threading
|
|
|
6
6
|
import time
|
|
7
7
|
from collections.abc import Mapping
|
|
8
8
|
from functools import lru_cache
|
|
9
|
-
from typing import Any, Self
|
|
9
|
+
from typing import Any, Self
|
|
10
10
|
|
|
11
11
|
import hvac
|
|
12
12
|
import requests
|
|
@@ -48,13 +48,6 @@ class VaultConnectionError(Exception):
|
|
|
48
48
|
pass
|
|
49
49
|
|
|
50
50
|
|
|
51
|
-
class Secret(TypedDict):
|
|
52
|
-
path: str
|
|
53
|
-
field: str
|
|
54
|
-
format: str | None
|
|
55
|
-
version: str | None
|
|
56
|
-
|
|
57
|
-
|
|
58
51
|
SECRET_VERSION_LATEST = "LATEST"
|
|
59
52
|
|
|
60
53
|
|
|
@@ -197,7 +190,7 @@ class VaultClient:
|
|
|
197
190
|
self._client.auth_approle(self.role_id, self.secret_id)
|
|
198
191
|
|
|
199
192
|
@retry()
|
|
200
|
-
def read_all_with_version(self, secret: Mapping) -> tuple[
|
|
193
|
+
def read_all_with_version(self, secret: Mapping) -> tuple[dict, int | None]:
|
|
201
194
|
"""Returns a dictionary of keys and values in a Vault secret and the
|
|
202
195
|
version of the secret, for V1 secrets, version will be None.
|
|
203
196
|
|
|
@@ -207,7 +200,7 @@ class VaultClient:
|
|
|
207
200
|
a v2 KV engine)
|
|
208
201
|
"""
|
|
209
202
|
secret_path = secret["path"]
|
|
210
|
-
secret_version = secret.get("version")
|
|
203
|
+
secret_version = secret.get("version") or SECRET_VERSION_LATEST
|
|
211
204
|
|
|
212
205
|
kv_version = self._get_mount_version_by_secret_path(secret_path)
|
|
213
206
|
|
|
@@ -250,7 +243,7 @@ class VaultClient:
|
|
|
250
243
|
|
|
251
244
|
def __read_all_v2(
|
|
252
245
|
self, path: str, version: str | None
|
|
253
|
-
) -> tuple[dict[str, Any],
|
|
246
|
+
) -> tuple[dict[str, Any], int]:
|
|
254
247
|
path_split = path.split("/")
|
|
255
248
|
mount_point = path_split[0]
|
|
256
249
|
read_path = "/".join(path_split[1:])
|
|
@@ -294,7 +287,7 @@ class VaultClient:
|
|
|
294
287
|
return secret["data"]
|
|
295
288
|
|
|
296
289
|
@retry()
|
|
297
|
-
def read(self, secret:
|
|
290
|
+
def read(self, secret: Mapping[str, Any]) -> Any:
|
|
298
291
|
"""Returns a value of a key in a Vault secret.
|
|
299
292
|
|
|
300
293
|
The input secret is a dictionary which contains the following fields:
|
reconcile/utils/vcs.py
CHANGED
|
@@ -140,7 +140,7 @@ class VCS:
|
|
|
140
140
|
gitlab_instances: Iterable[GitlabInstanceV1],
|
|
141
141
|
) -> GitLabApi:
|
|
142
142
|
return GitLabApi(
|
|
143
|
-
next(iter(gitlab_instances)).
|
|
143
|
+
next(iter(gitlab_instances)).model_dump(by_alias=True),
|
|
144
144
|
secret_reader=self._secret_reader,
|
|
145
145
|
)
|
|
146
146
|
|
|
@@ -150,7 +150,7 @@ class VCS:
|
|
|
150
150
|
app_interface_repo_url: str,
|
|
151
151
|
) -> GitLabApi:
|
|
152
152
|
return GitLabApi(
|
|
153
|
-
next(iter(gitlab_instances)).
|
|
153
|
+
next(iter(gitlab_instances)).model_dump(by_alias=True),
|
|
154
154
|
secret_reader=self._secret_reader,
|
|
155
155
|
project_url=app_interface_repo_url,
|
|
156
156
|
)
|
|
@@ -221,26 +221,26 @@ class VCS:
|
|
|
221
221
|
match repo_info.platform:
|
|
222
222
|
case "github":
|
|
223
223
|
github = self._init_github(repo_url=repo_url, auth_code=auth_code)
|
|
224
|
-
data = github.compare(commit_from=commit_from, commit_to=commit_to)
|
|
225
224
|
return [
|
|
226
225
|
Commit(
|
|
227
226
|
repo=repo_url,
|
|
228
227
|
sha=gh_commit.sha,
|
|
229
228
|
date=gh_commit.commit.committer.date,
|
|
230
229
|
)
|
|
231
|
-
for gh_commit in
|
|
230
|
+
for gh_commit in github.compare(
|
|
231
|
+
commit_from=commit_from, commit_to=commit_to
|
|
232
|
+
)
|
|
232
233
|
]
|
|
233
234
|
case "gitlab":
|
|
234
|
-
data = self._gitlab_instance.repository_compare(
|
|
235
|
-
repo_url=repo_url, ref_from=commit_from, ref_to=commit_to
|
|
236
|
-
)
|
|
237
235
|
return [
|
|
238
236
|
Commit(
|
|
239
237
|
repo=repo_url,
|
|
240
238
|
sha=gl_commit["id"],
|
|
241
239
|
date=datetime.fromisoformat(gl_commit["committed_date"]),
|
|
242
240
|
)
|
|
243
|
-
for gl_commit in
|
|
241
|
+
for gl_commit in self._gitlab_instance.repository_compare(
|
|
242
|
+
repo_url=repo_url, ref_from=commit_from, ref_to=commit_to
|
|
243
|
+
)
|
|
244
244
|
]
|
|
245
245
|
case _:
|
|
246
246
|
raise ValueError(f"Unsupported repository URL: {repo_url}")
|
reconcile/vault_replication.py
CHANGED
|
@@ -84,6 +84,54 @@ def deep_copy_versions(
|
|
|
84
84
|
dest_vault.write(secret=write_dict, decode_base64=False, force=True)
|
|
85
85
|
|
|
86
86
|
|
|
87
|
+
def _handle_missing_destination_secret(
|
|
88
|
+
dry_run: bool,
|
|
89
|
+
source_vault: VaultClient,
|
|
90
|
+
dest_vault: VaultClient,
|
|
91
|
+
source_data: dict,
|
|
92
|
+
source_version: int | None,
|
|
93
|
+
path: str,
|
|
94
|
+
) -> None:
|
|
95
|
+
"""Handles replication when destination secret is missing or has no accessible versions.
|
|
96
|
+
|
|
97
|
+
This covers two scenarios:
|
|
98
|
+
1. Secret doesn't exist at all in destination vault (SecretNotFoundError)
|
|
99
|
+
2. Secret exists but all versions are deleted in KV v2 (SecretVersionNotFoundError)
|
|
100
|
+
|
|
101
|
+
For both cases, we replicate from source starting from version 0 (or copy directly for v1).
|
|
102
|
+
|
|
103
|
+
Args:
|
|
104
|
+
dry_run: Whether this is a dry run
|
|
105
|
+
source_vault: Source vault client (needed for v2 deep copy)
|
|
106
|
+
dest_vault: Destination vault client
|
|
107
|
+
source_data: Already retrieved source secret data
|
|
108
|
+
source_version: Source secret version (None for v1 secrets)
|
|
109
|
+
path: Secret path
|
|
110
|
+
"""
|
|
111
|
+
if source_version is None:
|
|
112
|
+
# v1 secret - just copy it over using the already-retrieved source data
|
|
113
|
+
logging.info(["replicate_vault_secret", "Copying v1 secret", path])
|
|
114
|
+
if not dry_run:
|
|
115
|
+
write_dict = {"path": path, "data": source_data}
|
|
116
|
+
dest_vault.write(secret=write_dict, decode_base64=False, force=True)
|
|
117
|
+
else:
|
|
118
|
+
# v2 secret - deep copy all versions starting from 0
|
|
119
|
+
# Note: deep_copy_versions will read individual versions from source as needed
|
|
120
|
+
logging.info([
|
|
121
|
+
"replicate_vault_secret",
|
|
122
|
+
"Deep copying v2 secret versions",
|
|
123
|
+
path,
|
|
124
|
+
])
|
|
125
|
+
deep_copy_versions(
|
|
126
|
+
dry_run=dry_run,
|
|
127
|
+
source_vault=source_vault,
|
|
128
|
+
dest_vault=dest_vault,
|
|
129
|
+
current_dest_version=0,
|
|
130
|
+
current_source_version=source_version,
|
|
131
|
+
path=path,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
87
135
|
def write_dummy_versions(
|
|
88
136
|
dry_run: bool,
|
|
89
137
|
dest_vault: VaultClient,
|
|
@@ -133,48 +181,65 @@ def copy_vault_secret(
|
|
|
133
181
|
|
|
134
182
|
try:
|
|
135
183
|
dest_data, dest_version = dest_vault.read_all_with_version(secret_dict)
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
184
|
+
except SecretVersionNotFoundError:
|
|
185
|
+
# Handle KV v2 case where secret metadata exists but latest version is deleted
|
|
186
|
+
# This occurs when someone manually deletes the latest version but the secret
|
|
187
|
+
# metadata still exists in Vault. This should only happen for v2 secrets.
|
|
188
|
+
logging.info([
|
|
189
|
+
"replicate_vault_secret",
|
|
190
|
+
"KV v2 latest version deleted, replicating all versions",
|
|
191
|
+
path,
|
|
192
|
+
])
|
|
193
|
+
_handle_missing_destination_secret(
|
|
194
|
+
dry_run=dry_run,
|
|
195
|
+
source_vault=source_vault,
|
|
196
|
+
dest_vault=dest_vault,
|
|
197
|
+
source_data=source_data,
|
|
198
|
+
source_version=version,
|
|
199
|
+
path=path,
|
|
200
|
+
)
|
|
201
|
+
return
|
|
202
|
+
except SecretNotFoundError:
|
|
203
|
+
# Handle case where secret doesn't exist at all in destination vault
|
|
204
|
+
logging.info([
|
|
205
|
+
"replicate_vault_secret",
|
|
206
|
+
"Secret not found in destination",
|
|
207
|
+
path,
|
|
208
|
+
])
|
|
209
|
+
_handle_missing_destination_secret(
|
|
210
|
+
dry_run=dry_run,
|
|
211
|
+
source_vault=source_vault,
|
|
212
|
+
dest_vault=dest_vault,
|
|
213
|
+
source_data=source_data,
|
|
214
|
+
source_version=version,
|
|
215
|
+
path=path,
|
|
216
|
+
)
|
|
217
|
+
return
|
|
218
|
+
|
|
219
|
+
# If we reach here, we successfully read the destination secret
|
|
220
|
+
if dest_version is None or version is None:
|
|
221
|
+
# v1 secrets don't have version
|
|
222
|
+
if source_data == dest_data:
|
|
223
|
+
# If the secret is the same in both vaults, we don't need
|
|
224
|
+
# to copy it again
|
|
225
|
+
return
|
|
226
|
+
|
|
227
|
+
write_dict = {"path": path, "data": source_data}
|
|
228
|
+
logging.info(["replicate_vault_secret", path])
|
|
229
|
+
if not dry_run:
|
|
230
|
+
# Using force=True to write the secret to force the vault client even
|
|
231
|
+
# if the data is the same as the previous version. This happens in
|
|
232
|
+
# some secrets even tho the library does not create it
|
|
233
|
+
dest_vault.write(secret=write_dict, decode_base64=False, force=True)
|
|
234
|
+
elif dest_version < version:
|
|
235
|
+
deep_copy_versions(
|
|
236
|
+
dry_run=dry_run,
|
|
237
|
+
source_vault=source_vault,
|
|
238
|
+
dest_vault=dest_vault,
|
|
239
|
+
current_dest_version=dest_version,
|
|
240
|
+
current_source_version=version,
|
|
241
|
+
path=path,
|
|
242
|
+
)
|
|
178
243
|
|
|
179
244
|
|
|
180
245
|
def check_invalid_paths(
|
tools/app_interface_reporter.py
CHANGED
|
@@ -4,7 +4,6 @@ import os
|
|
|
4
4
|
import textwrap
|
|
5
5
|
from collections.abc import Mapping, MutableMapping
|
|
6
6
|
from datetime import (
|
|
7
|
-
UTC,
|
|
8
7
|
datetime,
|
|
9
8
|
)
|
|
10
9
|
|
|
@@ -29,6 +28,7 @@ from reconcile.cli import (
|
|
|
29
28
|
)
|
|
30
29
|
from reconcile.jenkins_job_builder import init_jjb
|
|
31
30
|
from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
|
|
31
|
+
from reconcile.utils.datetime_util import ensure_utc, utc_now
|
|
32
32
|
from reconcile.utils.mr import CreateAppInterfaceReporter
|
|
33
33
|
from reconcile.utils.runtime.environment import init_env
|
|
34
34
|
from reconcile.utils.secret_reader import SecretReader
|
|
@@ -189,8 +189,8 @@ def get_apps_data(
|
|
|
189
189
|
apps = queries.get_apps()
|
|
190
190
|
jjb = init_jjb(secret_reader)
|
|
191
191
|
jenkins_map = jenkins_base.get_jenkins_map()
|
|
192
|
-
time_limit = date - relativedelta(months=month_delta)
|
|
193
|
-
timestamp_limit = int(time_limit.
|
|
192
|
+
time_limit = ensure_utc(date) - relativedelta(months=month_delta)
|
|
193
|
+
timestamp_limit = int(time_limit.timestamp())
|
|
194
194
|
|
|
195
195
|
secret_content = secret_reader.read_all({"path": DASHDOTDB_SECRET})
|
|
196
196
|
dashdotdb_url = secret_content["url"]
|
|
@@ -411,7 +411,7 @@ def main(
|
|
|
411
411
|
) -> None:
|
|
412
412
|
init_env(log_level=log_level, config_file=configfile)
|
|
413
413
|
|
|
414
|
-
now =
|
|
414
|
+
now = utc_now()
|
|
415
415
|
apps = get_apps_data(now, thread_pool_size=thread_pool_size)
|
|
416
416
|
|
|
417
417
|
reports = [Report(app, now).to_message() for app in apps]
|