qontract-reconcile 0.10.2.dev310__py3-none-any.whl → 0.10.2.dev439__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of qontract-reconcile might be problematic. Click here for more details.
- {qontract_reconcile-0.10.2.dev310.dist-info → qontract_reconcile-0.10.2.dev439.dist-info}/METADATA +13 -12
- {qontract_reconcile-0.10.2.dev310.dist-info → qontract_reconcile-0.10.2.dev439.dist-info}/RECORD +396 -391
- reconcile/acs_rbac.py +2 -2
- reconcile/aus/advanced_upgrade_service.py +18 -12
- reconcile/aus/base.py +134 -32
- reconcile/aus/cluster_version_data.py +15 -5
- reconcile/aus/models.py +3 -1
- reconcile/aus/ocm_addons_upgrade_scheduler_org.py +1 -0
- reconcile/aus/ocm_upgrade_scheduler.py +8 -1
- reconcile/aus/ocm_upgrade_scheduler_org.py +20 -5
- reconcile/aus/version_gates/sts_version_gate_handler.py +54 -1
- reconcile/automated_actions/config/integration.py +16 -4
- reconcile/aws_account_manager/integration.py +8 -8
- reconcile/aws_account_manager/reconciler.py +3 -3
- reconcile/aws_ami_cleanup/integration.py +8 -12
- reconcile/aws_ami_share.py +69 -62
- reconcile/aws_cloudwatch_log_retention/integration.py +155 -126
- reconcile/aws_ecr_image_pull_secrets.py +5 -5
- reconcile/aws_iam_keys.py +1 -0
- reconcile/aws_saml_idp/integration.py +12 -4
- reconcile/aws_saml_roles/integration.py +32 -25
- reconcile/aws_version_sync/integration.py +125 -84
- reconcile/change_owners/bundle.py +3 -3
- reconcile/change_owners/change_log_tracking.py +3 -2
- reconcile/change_owners/change_owners.py +1 -1
- reconcile/change_owners/diff.py +2 -4
- reconcile/checkpoint.py +12 -4
- reconcile/cli.py +111 -18
- reconcile/cluster_deployment_mapper.py +2 -3
- reconcile/dashdotdb_dora.py +5 -12
- reconcile/dashdotdb_slo.py +1 -1
- reconcile/database_access_manager.py +125 -121
- reconcile/deadmanssnitch.py +1 -5
- reconcile/dynatrace_token_provider/integration.py +1 -1
- reconcile/endpoints_discovery/integration.py +4 -1
- reconcile/endpoints_discovery/merge_request.py +1 -1
- reconcile/endpoints_discovery/merge_request_manager.py +9 -11
- reconcile/external_resources/factories.py +5 -12
- reconcile/external_resources/integration.py +1 -1
- reconcile/external_resources/manager.py +8 -5
- reconcile/external_resources/meta.py +0 -1
- reconcile/external_resources/metrics.py +1 -1
- reconcile/external_resources/model.py +20 -20
- reconcile/external_resources/reconciler.py +7 -4
- reconcile/external_resources/secrets_sync.py +10 -14
- reconcile/external_resources/state.py +26 -16
- reconcile/fleet_labeler/integration.py +1 -1
- reconcile/gabi_authorized_users.py +8 -5
- reconcile/gcp_image_mirror.py +2 -2
- reconcile/github_org.py +1 -1
- reconcile/github_owners.py +4 -0
- reconcile/gitlab_housekeeping.py +13 -15
- reconcile/gitlab_members.py +6 -12
- reconcile/gitlab_mr_sqs_consumer.py +2 -2
- reconcile/gitlab_owners.py +15 -11
- reconcile/gitlab_permissions.py +8 -12
- reconcile/glitchtip_project_alerts/integration.py +3 -1
- reconcile/gql_definitions/acs/acs_instances.py +10 -10
- reconcile/gql_definitions/acs/acs_policies.py +5 -5
- reconcile/gql_definitions/acs/acs_rbac.py +6 -6
- reconcile/gql_definitions/advanced_upgrade_service/aus_clusters.py +32 -32
- reconcile/gql_definitions/advanced_upgrade_service/aus_organization.py +26 -26
- reconcile/gql_definitions/app_interface_metrics_exporter/onboarding_status.py +6 -7
- reconcile/gql_definitions/app_sre_tekton_access_revalidation/roles.py +5 -5
- reconcile/gql_definitions/app_sre_tekton_access_revalidation/users.py +5 -5
- reconcile/gql_definitions/automated_actions/instance.py +51 -12
- reconcile/gql_definitions/aws_account_manager/aws_accounts.py +11 -11
- reconcile/gql_definitions/aws_ami_cleanup/aws_accounts.py +20 -10
- reconcile/gql_definitions/aws_cloudwatch_log_retention/aws_accounts.py +28 -68
- reconcile/gql_definitions/aws_saml_idp/aws_accounts.py +20 -10
- reconcile/gql_definitions/aws_saml_roles/aws_accounts.py +20 -10
- reconcile/gql_definitions/aws_saml_roles/roles.py +5 -5
- reconcile/gql_definitions/aws_version_sync/clusters.py +10 -10
- reconcile/gql_definitions/aws_version_sync/namespaces.py +5 -5
- reconcile/gql_definitions/change_owners/queries/change_types.py +5 -5
- reconcile/gql_definitions/change_owners/queries/self_service_roles.py +9 -9
- reconcile/gql_definitions/cluster_auth_rhidp/clusters.py +18 -18
- reconcile/gql_definitions/common/alerting_services_settings.py +9 -9
- reconcile/gql_definitions/common/app_code_component_repos.py +5 -5
- reconcile/gql_definitions/common/app_interface_custom_messages.py +5 -5
- reconcile/gql_definitions/common/app_interface_dms_settings.py +5 -5
- reconcile/gql_definitions/common/app_interface_repo_settings.py +5 -5
- reconcile/gql_definitions/common/app_interface_roles.py +120 -0
- reconcile/gql_definitions/common/app_interface_state_settings.py +10 -10
- reconcile/gql_definitions/common/app_interface_vault_settings.py +5 -5
- reconcile/gql_definitions/common/app_quay_repos_escalation_policies.py +5 -5
- reconcile/gql_definitions/common/apps.py +5 -5
- reconcile/gql_definitions/common/aws_vpc_requests.py +22 -9
- reconcile/gql_definitions/common/aws_vpcs.py +11 -11
- reconcile/gql_definitions/common/clusters.py +37 -35
- reconcile/gql_definitions/common/clusters_minimal.py +14 -14
- reconcile/gql_definitions/common/clusters_with_dms.py +6 -6
- reconcile/gql_definitions/common/clusters_with_peering.py +29 -30
- reconcile/gql_definitions/common/github_orgs.py +10 -10
- reconcile/gql_definitions/common/jira_settings.py +10 -10
- reconcile/gql_definitions/common/jiralert_settings.py +5 -5
- reconcile/gql_definitions/common/ldap_settings.py +5 -5
- reconcile/gql_definitions/common/namespaces.py +42 -44
- reconcile/gql_definitions/common/namespaces_minimal.py +15 -13
- reconcile/gql_definitions/common/ocm_env_telemeter.py +12 -12
- reconcile/gql_definitions/common/ocm_environments.py +19 -19
- reconcile/gql_definitions/common/pagerduty_instances.py +9 -9
- reconcile/gql_definitions/common/pgp_reencryption_settings.py +6 -6
- reconcile/gql_definitions/common/pipeline_providers.py +29 -29
- reconcile/gql_definitions/common/quay_instances.py +5 -5
- reconcile/gql_definitions/common/quay_orgs.py +5 -5
- reconcile/gql_definitions/common/reserved_networks.py +5 -5
- reconcile/gql_definitions/common/rhcs_provider_settings.py +5 -5
- reconcile/gql_definitions/common/saas_files.py +44 -44
- reconcile/gql_definitions/common/saas_target_namespaces.py +10 -10
- reconcile/gql_definitions/common/saasherder_settings.py +5 -5
- reconcile/gql_definitions/common/slack_workspaces.py +5 -5
- reconcile/gql_definitions/common/smtp_client_settings.py +19 -19
- reconcile/gql_definitions/common/state_aws_account.py +7 -8
- reconcile/gql_definitions/common/users.py +5 -5
- reconcile/gql_definitions/common/users_with_paths.py +5 -5
- reconcile/gql_definitions/cost_report/app_names.py +5 -5
- reconcile/gql_definitions/cost_report/cost_namespaces.py +5 -5
- reconcile/gql_definitions/cost_report/settings.py +9 -9
- reconcile/gql_definitions/dashdotdb_slo/slo_documents_query.py +43 -43
- reconcile/gql_definitions/dynatrace_token_provider/dynatrace_bootstrap_tokens.py +10 -10
- reconcile/gql_definitions/dynatrace_token_provider/token_specs.py +5 -5
- reconcile/gql_definitions/email_sender/apps.py +5 -5
- reconcile/gql_definitions/email_sender/emails.py +8 -8
- reconcile/gql_definitions/email_sender/users.py +6 -6
- reconcile/gql_definitions/endpoints_discovery/apps.py +10 -10
- reconcile/gql_definitions/external_resources/aws_accounts.py +9 -9
- reconcile/gql_definitions/external_resources/external_resources_modules.py +23 -23
- reconcile/gql_definitions/external_resources/external_resources_namespaces.py +494 -410
- reconcile/gql_definitions/external_resources/external_resources_settings.py +28 -26
- reconcile/gql_definitions/external_resources/fragments/external_resources_module_overrides.py +5 -5
- reconcile/gql_definitions/fleet_labeler/fleet_labels.py +40 -40
- reconcile/gql_definitions/fragments/aus_organization.py +5 -5
- reconcile/gql_definitions/fragments/aws_account_common.py +7 -5
- reconcile/gql_definitions/fragments/aws_account_managed.py +5 -5
- reconcile/gql_definitions/fragments/aws_account_sso.py +5 -5
- reconcile/gql_definitions/fragments/aws_infra_management_account.py +5 -5
- reconcile/gql_definitions/fragments/{aws_vpc_request_subnet.py → aws_organization.py} +12 -8
- reconcile/gql_definitions/fragments/aws_vpc.py +5 -5
- reconcile/gql_definitions/fragments/aws_vpc_request.py +12 -5
- reconcile/gql_definitions/fragments/container_image_mirror.py +5 -5
- reconcile/gql_definitions/fragments/deploy_resources.py +5 -5
- reconcile/gql_definitions/fragments/disable.py +5 -5
- reconcile/gql_definitions/fragments/email_service.py +5 -5
- reconcile/gql_definitions/fragments/email_user.py +5 -5
- reconcile/gql_definitions/fragments/jumphost_common_fields.py +5 -5
- reconcile/gql_definitions/fragments/membership_source.py +5 -5
- reconcile/gql_definitions/fragments/minimal_ocm_organization.py +5 -5
- reconcile/gql_definitions/fragments/oc_connection_cluster.py +5 -5
- reconcile/gql_definitions/fragments/ocm_environment.py +5 -5
- reconcile/gql_definitions/fragments/pipeline_provider_retention.py +5 -5
- reconcile/gql_definitions/fragments/prometheus_instance.py +5 -5
- reconcile/gql_definitions/fragments/resource_limits_requirements.py +5 -5
- reconcile/gql_definitions/fragments/resource_requests_requirements.py +5 -5
- reconcile/gql_definitions/fragments/resource_values.py +5 -5
- reconcile/gql_definitions/fragments/saas_slo_document.py +5 -5
- reconcile/gql_definitions/fragments/saas_target_namespace.py +5 -5
- reconcile/gql_definitions/fragments/serviceaccount_token.py +5 -5
- reconcile/gql_definitions/fragments/terraform_state.py +5 -5
- reconcile/gql_definitions/fragments/upgrade_policy.py +5 -5
- reconcile/gql_definitions/fragments/user.py +5 -5
- reconcile/gql_definitions/fragments/vault_secret.py +5 -5
- reconcile/gql_definitions/gcp/gcp_docker_repos.py +9 -9
- reconcile/gql_definitions/gcp/gcp_projects.py +9 -9
- reconcile/gql_definitions/gitlab_members/gitlab_instances.py +9 -9
- reconcile/gql_definitions/gitlab_members/permissions.py +9 -9
- reconcile/gql_definitions/glitchtip/glitchtip_instance.py +9 -9
- reconcile/gql_definitions/glitchtip/glitchtip_project.py +11 -11
- reconcile/gql_definitions/glitchtip_project_alerts/glitchtip_project.py +9 -9
- reconcile/gql_definitions/integrations/integrations.py +48 -51
- reconcile/gql_definitions/introspection.json +3510 -1865
- reconcile/gql_definitions/jenkins_configs/jenkins_configs.py +11 -11
- reconcile/gql_definitions/jenkins_configs/jenkins_instances.py +10 -10
- reconcile/gql_definitions/jira/jira_servers.py +5 -5
- reconcile/gql_definitions/jira_permissions_validator/jira_boards_for_permissions_validator.py +14 -10
- reconcile/gql_definitions/jumphosts/jumphosts.py +13 -13
- reconcile/gql_definitions/ldap_groups/roles.py +5 -5
- reconcile/gql_definitions/ldap_groups/settings.py +9 -9
- reconcile/gql_definitions/maintenance/maintenances.py +5 -5
- reconcile/gql_definitions/membershipsources/roles.py +5 -5
- reconcile/gql_definitions/ocm_labels/clusters.py +18 -19
- reconcile/gql_definitions/ocm_labels/organizations.py +5 -5
- reconcile/gql_definitions/openshift_cluster_bots/clusters.py +22 -22
- reconcile/gql_definitions/openshift_groups/managed_groups.py +5 -5
- reconcile/gql_definitions/openshift_groups/managed_roles.py +6 -6
- reconcile/gql_definitions/openshift_serviceaccount_tokens/tokens.py +10 -10
- reconcile/gql_definitions/quay_membership/quay_membership.py +6 -6
- reconcile/gql_definitions/rhcs/certs.py +33 -87
- reconcile/gql_definitions/rhcs/openshift_resource_rhcs_cert.py +43 -0
- reconcile/gql_definitions/rhidp/organizations.py +18 -18
- reconcile/gql_definitions/service_dependencies/jenkins_instance_fragment.py +5 -5
- reconcile/gql_definitions/service_dependencies/service_dependencies.py +8 -8
- reconcile/gql_definitions/sharding/aws_accounts.py +10 -10
- reconcile/gql_definitions/sharding/ocm_organization.py +8 -8
- reconcile/gql_definitions/skupper_network/site_controller_template.py +5 -5
- reconcile/gql_definitions/skupper_network/skupper_networks.py +10 -10
- reconcile/gql_definitions/slack_usergroups/clusters.py +5 -5
- reconcile/gql_definitions/slack_usergroups/permissions.py +9 -9
- reconcile/gql_definitions/slack_usergroups/users.py +5 -5
- reconcile/gql_definitions/slo_documents/slo_documents.py +5 -5
- reconcile/gql_definitions/status_board/status_board.py +6 -7
- reconcile/gql_definitions/statuspage/statuspages.py +9 -9
- reconcile/gql_definitions/templating/template_collection.py +5 -5
- reconcile/gql_definitions/templating/templates.py +5 -5
- reconcile/gql_definitions/terraform_cloudflare_dns/app_interface_cloudflare_dns_settings.py +6 -6
- reconcile/gql_definitions/terraform_cloudflare_dns/terraform_cloudflare_zones.py +11 -11
- reconcile/gql_definitions/terraform_cloudflare_resources/terraform_cloudflare_accounts.py +11 -11
- reconcile/gql_definitions/terraform_cloudflare_resources/terraform_cloudflare_resources.py +20 -25
- reconcile/gql_definitions/terraform_cloudflare_users/app_interface_setting_cloudflare_and_vault.py +6 -6
- reconcile/gql_definitions/terraform_cloudflare_users/terraform_cloudflare_roles.py +12 -12
- reconcile/gql_definitions/terraform_init/aws_accounts.py +23 -9
- reconcile/gql_definitions/terraform_repo/terraform_repo.py +9 -9
- reconcile/gql_definitions/terraform_resources/database_access_manager.py +5 -5
- reconcile/gql_definitions/terraform_resources/terraform_resources_namespaces.py +450 -402
- reconcile/gql_definitions/terraform_tgw_attachments/aws_accounts.py +23 -17
- reconcile/gql_definitions/unleash_feature_toggles/feature_toggles.py +9 -9
- reconcile/gql_definitions/vault_instances/vault_instances.py +61 -61
- reconcile/gql_definitions/vault_policies/vault_policies.py +11 -11
- reconcile/gql_definitions/vpc_peerings_validator/vpc_peerings_validator.py +8 -8
- reconcile/gql_definitions/vpc_peerings_validator/vpc_peerings_validator_peered_cluster_fragment.py +5 -5
- reconcile/integrations_manager.py +3 -3
- reconcile/jenkins_job_builder.py +1 -1
- reconcile/jenkins_worker_fleets.py +80 -11
- reconcile/jira_permissions_validator.py +237 -122
- reconcile/ldap_groups/integration.py +1 -1
- reconcile/ocm/types.py +35 -56
- reconcile/ocm_aws_infrastructure_access.py +1 -1
- reconcile/ocm_clusters.py +4 -4
- reconcile/ocm_labels/integration.py +3 -2
- reconcile/ocm_machine_pools.py +33 -27
- reconcile/openshift_base.py +122 -10
- reconcile/openshift_cluster_bots.py +5 -5
- reconcile/openshift_groups.py +5 -0
- reconcile/openshift_limitranges.py +1 -1
- reconcile/openshift_namespace_labels.py +1 -1
- reconcile/openshift_namespaces.py +97 -101
- reconcile/openshift_resources_base.py +10 -5
- reconcile/openshift_rhcs_certs.py +77 -40
- reconcile/openshift_rolebindings.py +230 -130
- reconcile/openshift_saas_deploy.py +6 -7
- reconcile/openshift_saas_deploy_change_tester.py +9 -7
- reconcile/openshift_saas_deploy_trigger_cleaner.py +3 -5
- reconcile/openshift_serviceaccount_tokens.py +8 -7
- reconcile/openshift_tekton_resources.py +1 -1
- reconcile/openshift_upgrade_watcher.py +4 -4
- reconcile/openshift_users.py +5 -3
- reconcile/oum/labelset.py +5 -3
- reconcile/oum/models.py +1 -4
- reconcile/oum/providers.py +1 -1
- reconcile/prometheus_rules_tester/integration.py +4 -4
- reconcile/quay_mirror.py +1 -1
- reconcile/queries.py +131 -0
- reconcile/requests_sender.py +8 -3
- reconcile/resource_scraper.py +1 -5
- reconcile/rhidp/common.py +3 -5
- reconcile/rhidp/sso_client/base.py +19 -10
- reconcile/saas_auto_promotions_manager/merge_request_manager/renderer.py +1 -1
- reconcile/saas_auto_promotions_manager/subscriber.py +4 -3
- reconcile/sendgrid_teammates.py +20 -9
- reconcile/skupper_network/integration.py +2 -2
- reconcile/slack_usergroups.py +35 -14
- reconcile/sql_query.py +1 -0
- reconcile/status.py +2 -2
- reconcile/status_board.py +6 -6
- reconcile/statuspage/atlassian.py +7 -7
- reconcile/statuspage/integrations/maintenances.py +4 -3
- reconcile/statuspage/page.py +4 -9
- reconcile/statuspage/status.py +5 -8
- reconcile/templates/rosa-classic-cluster-creation.sh.j2 +5 -1
- reconcile/templates/rosa-hcp-cluster-creation.sh.j2 +4 -1
- reconcile/templating/lib/merge_request_manager.py +2 -2
- reconcile/templating/lib/rendering.py +3 -3
- reconcile/templating/renderer.py +12 -13
- reconcile/terraform_aws_route53.py +18 -8
- reconcile/terraform_cloudflare_dns.py +3 -3
- reconcile/terraform_cloudflare_resources.py +12 -13
- reconcile/terraform_cloudflare_users.py +3 -2
- reconcile/terraform_init/integration.py +187 -23
- reconcile/terraform_repo.py +16 -12
- reconcile/terraform_resources.py +18 -10
- reconcile/terraform_tgw_attachments.py +28 -20
- reconcile/terraform_users.py +27 -22
- reconcile/terraform_vpc_peerings.py +15 -3
- reconcile/terraform_vpc_resources/integration.py +23 -8
- reconcile/typed_queries/app_interface_roles.py +10 -0
- reconcile/typed_queries/aws_account_tags.py +41 -0
- reconcile/typed_queries/cost_report/app_names.py +1 -1
- reconcile/typed_queries/cost_report/cost_namespaces.py +2 -2
- reconcile/typed_queries/saas_files.py +13 -13
- reconcile/typed_queries/status_board.py +2 -2
- reconcile/unleash_feature_toggles/integration.py +4 -2
- reconcile/utils/acs/base.py +6 -3
- reconcile/utils/acs/policies.py +2 -2
- reconcile/utils/aggregated_list.py +4 -3
- reconcile/utils/aws_api.py +51 -20
- reconcile/utils/aws_api_typed/api.py +38 -9
- reconcile/utils/aws_api_typed/cloudformation.py +149 -0
- reconcile/utils/aws_api_typed/logs.py +73 -0
- reconcile/utils/aws_api_typed/organization.py +4 -2
- reconcile/utils/binary.py +7 -12
- reconcile/utils/datetime_util.py +67 -0
- reconcile/utils/deadmanssnitch_api.py +1 -1
- reconcile/utils/differ.py +2 -3
- reconcile/utils/early_exit_cache.py +11 -12
- reconcile/utils/expiration.py +7 -3
- reconcile/utils/external_resource_spec.py +24 -1
- reconcile/utils/filtering.py +1 -1
- reconcile/utils/gitlab_api.py +7 -5
- reconcile/utils/glitchtip/client.py +6 -2
- reconcile/utils/glitchtip/models.py +25 -28
- reconcile/utils/gpg.py +5 -3
- reconcile/utils/gql.py +4 -7
- reconcile/utils/helm.py +2 -1
- reconcile/utils/helpers.py +1 -1
- reconcile/utils/imap_client.py +1 -1
- reconcile/utils/instrumented_wrappers.py +1 -1
- reconcile/utils/internal_groups/client.py +2 -2
- reconcile/utils/internal_groups/models.py +8 -17
- reconcile/utils/jenkins_api.py +24 -1
- reconcile/utils/jinja2/utils.py +6 -8
- reconcile/utils/jira_client.py +82 -63
- reconcile/utils/jjb_client.py +78 -46
- reconcile/utils/jobcontroller/controller.py +2 -2
- reconcile/utils/jobcontroller/models.py +17 -1
- reconcile/utils/json.py +74 -0
- reconcile/utils/ldap_client.py +4 -3
- reconcile/utils/lean_terraform_client.py +3 -1
- reconcile/utils/membershipsources/app_interface_resolver.py +4 -2
- reconcile/utils/membershipsources/models.py +16 -23
- reconcile/utils/membershipsources/resolver.py +4 -2
- reconcile/utils/merge_request_manager/merge_request_manager.py +4 -4
- reconcile/utils/merge_request_manager/parser.py +6 -6
- reconcile/utils/metrics.py +5 -5
- reconcile/utils/models.py +304 -82
- reconcile/utils/mr/__init__.py +3 -1
- reconcile/utils/mr/app_interface_reporter.py +6 -3
- reconcile/utils/mr/aws_access.py +1 -1
- reconcile/utils/mr/base.py +7 -13
- reconcile/utils/mr/clusters_updates.py +4 -2
- reconcile/utils/mr/notificator.py +3 -3
- reconcile/utils/mr/ocm_upgrade_scheduler_org_updates.py +4 -1
- reconcile/utils/mr/promote_qontract.py +28 -12
- reconcile/utils/mr/update_access_report_base.py +3 -4
- reconcile/utils/mr/user_maintenance.py +7 -6
- reconcile/utils/oc.py +445 -336
- reconcile/utils/oc_filters.py +3 -3
- reconcile/utils/ocm/addons.py +0 -1
- reconcile/utils/ocm/base.py +18 -21
- reconcile/utils/ocm/cluster_groups.py +1 -1
- reconcile/utils/ocm/identity_providers.py +2 -2
- reconcile/utils/ocm/labels.py +1 -1
- reconcile/utils/ocm/ocm.py +81 -71
- reconcile/utils/ocm/products.py +9 -3
- reconcile/utils/ocm/search_filters.py +3 -6
- reconcile/utils/ocm/service_log.py +4 -6
- reconcile/utils/ocm/sre_capability_labels.py +20 -13
- reconcile/utils/ocm_base_client.py +4 -4
- reconcile/utils/openshift_resource.py +83 -52
- reconcile/utils/openssl.py +2 -2
- reconcile/utils/output.py +3 -2
- reconcile/utils/pagerduty_api.py +10 -7
- reconcile/utils/promotion_state.py +6 -11
- reconcile/utils/raw_github_api.py +11 -8
- reconcile/utils/repo_owners.py +21 -29
- reconcile/utils/rhcsv2_certs.py +138 -35
- reconcile/utils/rosa/session.py +16 -0
- reconcile/utils/runtime/integration.py +2 -3
- reconcile/utils/runtime/meta.py +2 -1
- reconcile/utils/runtime/runner.py +2 -2
- reconcile/utils/saasherder/interfaces.py +13 -20
- reconcile/utils/saasherder/models.py +25 -21
- reconcile/utils/saasherder/saasherder.py +60 -32
- reconcile/utils/secret_reader.py +6 -6
- reconcile/utils/sharding.py +1 -1
- reconcile/utils/slack_api.py +26 -4
- reconcile/utils/sloth.py +224 -0
- reconcile/utils/sqs_gateway.py +16 -11
- reconcile/utils/state.py +2 -1
- reconcile/utils/structs.py +1 -1
- reconcile/utils/terraform_client.py +29 -26
- reconcile/utils/terrascript_aws_client.py +200 -116
- reconcile/utils/three_way_diff_strategy.py +1 -1
- reconcile/utils/unleash/server.py +2 -8
- reconcile/utils/vault.py +44 -41
- reconcile/utils/vcs.py +8 -8
- reconcile/vault_replication.py +119 -58
- tools/app_interface_reporter.py +4 -4
- tools/cli_commands/cost_report/cost_management_api.py +3 -3
- tools/cli_commands/cost_report/view.py +7 -6
- tools/cli_commands/erv2.py +1 -1
- tools/cli_commands/gpg_encrypt.py +4 -1
- tools/cli_commands/systems_and_tools.py +5 -1
- tools/qontract_cli.py +36 -21
- tools/template_validation.py +3 -1
- reconcile/gql_definitions/ocm_oidc_idp/__init__.py +0 -0
- reconcile/gql_definitions/ocm_subscription_labels/__init__.py +0 -0
- reconcile/jenkins/__init__.py +0 -0
- reconcile/jenkins/types.py +0 -77
- {qontract_reconcile-0.10.2.dev310.dist-info → qontract_reconcile-0.10.2.dev439.dist-info}/WHEEL +0 -0
- {qontract_reconcile-0.10.2.dev310.dist-info → qontract_reconcile-0.10.2.dev439.dist-info}/entry_points.txt +0 -0
|
@@ -1,8 +1,4 @@
|
|
|
1
1
|
import hashlib
|
|
2
|
-
import json
|
|
3
|
-
from abc import (
|
|
4
|
-
ABC,
|
|
5
|
-
)
|
|
6
2
|
from collections.abc import ItemsView, Iterable, Iterator, MutableMapping
|
|
7
3
|
from enum import StrEnum
|
|
8
4
|
from typing import Any
|
|
@@ -23,6 +19,7 @@ from reconcile.gql_definitions.external_resources.external_resources_namespaces
|
|
|
23
19
|
NamespaceTerraformResourceElastiCacheV1,
|
|
24
20
|
NamespaceTerraformResourceKMSV1,
|
|
25
21
|
NamespaceTerraformResourceMskV1,
|
|
22
|
+
NamespaceTerraformResourceRDSProxyV1,
|
|
26
23
|
NamespaceTerraformResourceRDSV1,
|
|
27
24
|
NamespaceV1,
|
|
28
25
|
)
|
|
@@ -37,6 +34,7 @@ from reconcile.utils.exceptions import FetchResourceError
|
|
|
37
34
|
from reconcile.utils.external_resource_spec import (
|
|
38
35
|
ExternalResourceSpec,
|
|
39
36
|
)
|
|
37
|
+
from reconcile.utils.json import json_dumps
|
|
40
38
|
|
|
41
39
|
|
|
42
40
|
class ExternalResourceOrphanedResourcesError(Exception):
|
|
@@ -87,11 +85,6 @@ class ExternalResourceKey(BaseModel, frozen=True):
|
|
|
87
85
|
provider=spec.provider,
|
|
88
86
|
)
|
|
89
87
|
|
|
90
|
-
def hash(self) -> str:
|
|
91
|
-
return hashlib.md5(
|
|
92
|
-
json.dumps(self.dict(), sort_keys=True).encode("utf-8")
|
|
93
|
-
).hexdigest()
|
|
94
|
-
|
|
95
88
|
@property
|
|
96
89
|
def state_path(self) -> str:
|
|
97
90
|
return f"{self.provision_provider}/{self.provisioner_name}/{self.provider}/{self.identifier}"
|
|
@@ -104,6 +97,7 @@ SUPPORTED_RESOURCE_TYPES = (
|
|
|
104
97
|
| NamespaceTerraformResourceElastiCacheV1
|
|
105
98
|
| NamespaceTerraformResourceKMSV1
|
|
106
99
|
| NamespaceTerraformResourceCloudWatchV1
|
|
100
|
+
| NamespaceTerraformResourceRDSProxyV1
|
|
107
101
|
)
|
|
108
102
|
|
|
109
103
|
|
|
@@ -115,7 +109,9 @@ class ExternalResourcesInventory(MutableMapping):
|
|
|
115
109
|
(rp, ns)
|
|
116
110
|
for ns in namespaces
|
|
117
111
|
for rp in ns.external_resources or []
|
|
118
|
-
if isinstance(rp, SUPPORTED_RESOURCE_PROVIDERS)
|
|
112
|
+
if isinstance(rp, SUPPORTED_RESOURCE_PROVIDERS)
|
|
113
|
+
and rp.resources
|
|
114
|
+
and ns.managed_external_resources
|
|
119
115
|
]
|
|
120
116
|
|
|
121
117
|
desired_specs = [
|
|
@@ -136,17 +132,17 @@ class ExternalResourcesInventory(MutableMapping):
|
|
|
136
132
|
) -> ExternalResourceSpec:
|
|
137
133
|
spec = ExternalResourceSpec(
|
|
138
134
|
provision_provider=provider.provider,
|
|
139
|
-
provisioner=provider.provisioner.
|
|
140
|
-
resource=resource.
|
|
135
|
+
provisioner=provider.provisioner.model_dump(),
|
|
136
|
+
resource=resource.model_dump(
|
|
141
137
|
exclude={
|
|
142
138
|
FLAG_RESOURCE_MANAGED_BY_ERV2,
|
|
143
139
|
FLAG_DELETE_RESOURCE,
|
|
144
140
|
MODULE_OVERRIDES,
|
|
145
141
|
}
|
|
146
142
|
),
|
|
147
|
-
namespace=namespace.
|
|
143
|
+
namespace=namespace.model_dump(by_alias=True),
|
|
148
144
|
)
|
|
149
|
-
spec.metadata[FLAG_DELETE_RESOURCE] = resource.delete
|
|
145
|
+
spec.metadata[FLAG_DELETE_RESOURCE] = resource.delete or namespace.delete
|
|
150
146
|
spec.metadata[MODULE_OVERRIDES] = resource.module_overrides
|
|
151
147
|
return spec
|
|
152
148
|
|
|
@@ -385,7 +381,7 @@ class Reconciliation(BaseModel, frozen=True):
|
|
|
385
381
|
)
|
|
386
382
|
# linked_resources store dependants resources. They will get reconciled
|
|
387
383
|
# every time the parent resource reconciliation finishes.
|
|
388
|
-
linked_resources: frozenset[ExternalResourceKey] | None
|
|
384
|
+
linked_resources: frozenset[ExternalResourceKey] | None = None
|
|
389
385
|
|
|
390
386
|
|
|
391
387
|
class ReconcileAction(StrEnum):
|
|
@@ -405,7 +401,7 @@ class ReconciliationStatus(BaseModel):
|
|
|
405
401
|
resource_status: ResourceStatus
|
|
406
402
|
|
|
407
403
|
|
|
408
|
-
class ModuleProvisionData(
|
|
404
|
+
class ModuleProvisionData(BaseModel):
|
|
409
405
|
pass
|
|
410
406
|
|
|
411
407
|
|
|
@@ -430,7 +426,7 @@ class ExternalResourceProvision(BaseModel):
|
|
|
430
426
|
target_cluster: str
|
|
431
427
|
target_namespace: str
|
|
432
428
|
target_secret_name: str
|
|
433
|
-
module_provision_data: ModuleProvisionData
|
|
429
|
+
module_provision_data: ModuleProvisionData | TerraformModuleProvisionData
|
|
434
430
|
|
|
435
431
|
|
|
436
432
|
class ExternalResource(BaseModel):
|
|
@@ -438,6 +434,10 @@ class ExternalResource(BaseModel):
|
|
|
438
434
|
provision: ExternalResourceProvision
|
|
439
435
|
|
|
440
436
|
def hash(self) -> str:
|
|
441
|
-
return hashlib.
|
|
442
|
-
|
|
443
|
-
|
|
437
|
+
return hashlib.sha256(json_dumps(self.data).encode("utf-8")).hexdigest()
|
|
438
|
+
|
|
439
|
+
def export(
|
|
440
|
+
self, exclude: dict[str, Any] | None = None, indent: int | None = None
|
|
441
|
+
) -> str:
|
|
442
|
+
"""Export the ExternalResource as a JSON string."""
|
|
443
|
+
return json_dumps(self, exclude=exclude, indent=indent)
|
|
@@ -70,10 +70,13 @@ class ReconciliationK8sJob(K8sJob, BaseModel, frozen=True):
|
|
|
70
70
|
dry_run_suffix: str = ""
|
|
71
71
|
|
|
72
72
|
def name_prefix(self) -> str:
|
|
73
|
+
identifier = (
|
|
74
|
+
f"{self.reconciliation.key.provider}-{self.reconciliation.key.identifier}"
|
|
75
|
+
)
|
|
73
76
|
if self.is_dry_run:
|
|
74
|
-
return f"er-dry-run-mr-{self.dry_run_suffix}"
|
|
77
|
+
return f"er-dry-run-mr-{self.dry_run_suffix}-{identifier}"
|
|
75
78
|
else:
|
|
76
|
-
return "er"
|
|
79
|
+
return f"er-{identifier}"
|
|
77
80
|
|
|
78
81
|
def unit_of_work_identity(self) -> Any:
|
|
79
82
|
return self.reconciliation.key
|
|
@@ -96,10 +99,10 @@ class ReconciliationK8sJob(K8sJob, BaseModel, frozen=True):
|
|
|
96
99
|
image=self.reconciliation.module_configuration.image_version,
|
|
97
100
|
image_pull_policy="Always",
|
|
98
101
|
resources=V1ResourceRequirements(
|
|
99
|
-
requests=self.reconciliation.module_configuration.resources.requests.
|
|
102
|
+
requests=self.reconciliation.module_configuration.resources.requests.model_dump(
|
|
100
103
|
exclude_none=True
|
|
101
104
|
),
|
|
102
|
-
limits=self.reconciliation.module_configuration.resources.limits.
|
|
105
|
+
limits=self.reconciliation.module_configuration.resources.limits.model_dump(
|
|
103
106
|
exclude_none=True
|
|
104
107
|
),
|
|
105
108
|
),
|
|
@@ -3,7 +3,6 @@ import json
|
|
|
3
3
|
import logging
|
|
4
4
|
from abc import abstractmethod
|
|
5
5
|
from collections.abc import Iterable, Mapping
|
|
6
|
-
from datetime import UTC, datetime
|
|
7
6
|
from hashlib import shake_128
|
|
8
7
|
from typing import Any
|
|
9
8
|
|
|
@@ -18,17 +17,18 @@ from reconcile.external_resources.meta import (
|
|
|
18
17
|
SECRET_ANN_PROVISION_PROVIDER,
|
|
19
18
|
SECRET_ANN_PROVISIONER,
|
|
20
19
|
SECRET_UPDATED_AT,
|
|
21
|
-
SECRET_UPDATED_AT_TIMEFORMAT,
|
|
22
20
|
)
|
|
23
21
|
from reconcile.external_resources.model import (
|
|
24
22
|
ExternalResourceKey,
|
|
25
23
|
)
|
|
26
24
|
from reconcile.openshift_base import ApplyOptions, apply_action
|
|
27
25
|
from reconcile.typed_queries.clusters_minimal import get_clusters_minimal
|
|
26
|
+
from reconcile.utils.datetime_util import to_utc_seconds_iso_format, utc_now
|
|
28
27
|
from reconcile.utils.differ import diff_mappings
|
|
29
28
|
from reconcile.utils.external_resource_spec import (
|
|
30
29
|
ExternalResourceSpec,
|
|
31
30
|
)
|
|
31
|
+
from reconcile.utils.json import json_dumps
|
|
32
32
|
from reconcile.utils.oc import (
|
|
33
33
|
OCCli,
|
|
34
34
|
)
|
|
@@ -38,7 +38,6 @@ from reconcile.utils.secret_reader import SecretNotFoundError, SecretReaderBase
|
|
|
38
38
|
from reconcile.utils.three_way_diff_strategy import three_way_diff_using_hash
|
|
39
39
|
from reconcile.utils.vault import (
|
|
40
40
|
VaultClient,
|
|
41
|
-
_VaultClient, # noqa
|
|
42
41
|
)
|
|
43
42
|
|
|
44
43
|
|
|
@@ -47,8 +46,8 @@ class VaultSecret(BaseModel):
|
|
|
47
46
|
|
|
48
47
|
path: str
|
|
49
48
|
field: str
|
|
50
|
-
version: int | None
|
|
51
|
-
q_format: str | None
|
|
49
|
+
version: int | None = None
|
|
50
|
+
q_format: str | None = None
|
|
52
51
|
|
|
53
52
|
|
|
54
53
|
class SecretHelper:
|
|
@@ -155,7 +154,7 @@ class SecretsReconciler:
|
|
|
155
154
|
annotations[SECRET_ANN_PROVIDER] = spec.provider
|
|
156
155
|
annotations[SECRET_ANN_IDENTIFIER] = spec.identifier
|
|
157
156
|
annotations[SECRET_UPDATED_AT] = spec.metadata[SECRET_UPDATED_AT]
|
|
158
|
-
spec.resource["annotations"] =
|
|
157
|
+
spec.resource["annotations"] = json_dumps(annotations)
|
|
159
158
|
|
|
160
159
|
def _specs_with_secret(
|
|
161
160
|
self,
|
|
@@ -352,9 +351,7 @@ class InClusterSecretsReconciler(SecretsReconciler):
|
|
|
352
351
|
secret_name = secret["metadata"]["name"]
|
|
353
352
|
spec = secrets_map[secret_name]
|
|
354
353
|
spec.secret = self.output_secrets_formatter.format(secret["data"])
|
|
355
|
-
spec.metadata[SECRET_UPDATED_AT] =
|
|
356
|
-
SECRET_UPDATED_AT_TIMEFORMAT
|
|
357
|
-
)
|
|
354
|
+
spec.metadata[SECRET_UPDATED_AT] = to_utc_seconds_iso_format(utc_now())
|
|
358
355
|
|
|
359
356
|
def _delete_source_secret(self, spec: ExternalResourceSpec) -> None:
|
|
360
357
|
secret_name = self._get_spec_outputs_secret_name(spec)
|
|
@@ -368,7 +365,7 @@ class InClusterSecretsReconciler(SecretsReconciler):
|
|
|
368
365
|
"path": self.secret_path(self.vault_path, spec),
|
|
369
366
|
"data": secret,
|
|
370
367
|
}
|
|
371
|
-
self.vault_client.write(desired_secret, decode_base64=False)
|
|
368
|
+
self.vault_client.write(desired_secret, decode_base64=False)
|
|
372
369
|
|
|
373
370
|
def sync_secrets(
|
|
374
371
|
self, specs: Iterable[ExternalResourceSpec]
|
|
@@ -423,7 +420,7 @@ def build_incluster_secrets_reconciler(
|
|
|
423
420
|
ri=ri,
|
|
424
421
|
oc=oc,
|
|
425
422
|
vault_path=vault_path,
|
|
426
|
-
vault_client=VaultClient(),
|
|
423
|
+
vault_client=VaultClient.get_instance(),
|
|
427
424
|
secrets_reader=secrets_reader,
|
|
428
425
|
output_secrets_formatter=OutputSecretsFormatter(secrets_reader),
|
|
429
426
|
thread_pool_size=thread_pool_size,
|
|
@@ -451,9 +448,8 @@ class VaultSecretsReconciler(SecretsReconciler):
|
|
|
451
448
|
secret_path = self.secret_path(self.vault_path, spec)
|
|
452
449
|
try:
|
|
453
450
|
logging.debug("Reading Secret %s", secret_path)
|
|
454
|
-
data = self.secrets_reader.read_all({"path": secret_path})
|
|
455
|
-
spec.metadata[SECRET_UPDATED_AT] = data
|
|
456
|
-
del data[SECRET_UPDATED_AT]
|
|
451
|
+
data = self.secrets_reader.read_all({"path": secret_path}).copy()
|
|
452
|
+
spec.metadata[SECRET_UPDATED_AT] = data.pop(SECRET_UPDATED_AT)
|
|
457
453
|
spec.secret = data
|
|
458
454
|
except SecretNotFoundError:
|
|
459
455
|
logging.info("Error getting secret from vault, skipping. [%s]", secret_path)
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import json
|
|
1
2
|
import logging
|
|
2
3
|
from collections.abc import Mapping
|
|
3
|
-
from datetime import
|
|
4
|
+
from datetime import datetime
|
|
4
5
|
from enum import StrEnum
|
|
6
|
+
from hashlib import sha256
|
|
5
7
|
from typing import Any
|
|
6
8
|
|
|
7
9
|
from pydantic import BaseModel
|
|
@@ -16,6 +18,8 @@ from reconcile.external_resources.model import (
|
|
|
16
18
|
ResourceStatus,
|
|
17
19
|
)
|
|
18
20
|
from reconcile.utils.aws_api_typed.api import AWSApi
|
|
21
|
+
from reconcile.utils.datetime_util import to_utc_microseconds_iso_format, utc_now
|
|
22
|
+
from reconcile.utils.json import json_dumps
|
|
19
23
|
|
|
20
24
|
DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
|
|
21
25
|
|
|
@@ -41,7 +45,7 @@ class ExternalResourceState(BaseModel):
|
|
|
41
45
|
self, reconciliation_status: ReconciliationStatus
|
|
42
46
|
) -> None:
|
|
43
47
|
if self.reconciliation_needs_state_update(reconciliation_status):
|
|
44
|
-
self.ts =
|
|
48
|
+
self.ts = utc_now()
|
|
45
49
|
self.resource_status = reconciliation_status.resource_status
|
|
46
50
|
|
|
47
51
|
def reconciliation_needs_state_update(
|
|
@@ -169,8 +173,8 @@ class DynamoDBStateAdapter:
|
|
|
169
173
|
|
|
170
174
|
def serialize(self, state: ExternalResourceState) -> dict[str, Any]:
|
|
171
175
|
return {
|
|
172
|
-
self.ER_KEY_HASH: {"S": state.key.
|
|
173
|
-
self.TIMESTAMP: {"S": state.ts
|
|
176
|
+
self.ER_KEY_HASH: {"S": state.key.state_path},
|
|
177
|
+
self.TIMESTAMP: {"S": to_utc_microseconds_iso_format(state.ts)},
|
|
174
178
|
self.RESOURCE_STATUS: {"S": state.resource_status.value},
|
|
175
179
|
self.ER_KEY: {
|
|
176
180
|
"M": {
|
|
@@ -258,24 +262,30 @@ class ExternalResourcesStateDynamoDB:
|
|
|
258
262
|
self._table = table_name
|
|
259
263
|
self.partial_resources = self._get_partial_resources()
|
|
260
264
|
|
|
265
|
+
def _new_sha256_hash(self, item: dict) -> str:
|
|
266
|
+
resource_json = item[self.adapter.RECONC]["M"][self.adapter.RECONC_INPUT]["S"]
|
|
267
|
+
resource_dict = json.loads(resource_json)
|
|
268
|
+
data = resource_dict["data"]
|
|
269
|
+
return sha256(json_dumps(data).encode("utf-8")).hexdigest()
|
|
270
|
+
|
|
261
271
|
def get_external_resource_state(
|
|
262
|
-
self,
|
|
272
|
+
self,
|
|
273
|
+
key: ExternalResourceKey,
|
|
263
274
|
) -> ExternalResourceState:
|
|
264
275
|
data = self.aws_api.dynamodb.boto3_client.get_item(
|
|
265
276
|
TableName=self._table,
|
|
266
277
|
ConsistentRead=True,
|
|
267
|
-
Key={self.adapter.ER_KEY_HASH: {"S": key.
|
|
278
|
+
Key={self.adapter.ER_KEY_HASH: {"S": key.state_path}},
|
|
268
279
|
)
|
|
269
280
|
if "Item" in data:
|
|
270
281
|
return self.adapter.deserialize(data["Item"])
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
)
|
|
282
|
+
return ExternalResourceState(
|
|
283
|
+
key=key,
|
|
284
|
+
ts=utc_now(),
|
|
285
|
+
resource_status=ResourceStatus.NOT_EXISTS,
|
|
286
|
+
reconciliation=Reconciliation(key=key),
|
|
287
|
+
reconciliation_errors=0,
|
|
288
|
+
)
|
|
279
289
|
|
|
280
290
|
def set_external_resource_state(
|
|
281
291
|
self,
|
|
@@ -288,7 +298,7 @@ class ExternalResourcesStateDynamoDB:
|
|
|
288
298
|
def del_external_resource_state(self, key: ExternalResourceKey) -> None:
|
|
289
299
|
self.aws_api.dynamodb.boto3_client.delete_item(
|
|
290
300
|
TableName=self._table,
|
|
291
|
-
Key={self.adapter.ER_KEY_HASH: {"S": key.
|
|
301
|
+
Key={self.adapter.ER_KEY_HASH: {"S": key.state_path}},
|
|
292
302
|
)
|
|
293
303
|
|
|
294
304
|
def _get_partial_resources(
|
|
@@ -330,7 +340,7 @@ class ExternalResourcesStateDynamoDB:
|
|
|
330
340
|
) -> None:
|
|
331
341
|
self.aws_api.dynamodb.boto3_client.update_item(
|
|
332
342
|
TableName=self._table,
|
|
333
|
-
Key={self.adapter.ER_KEY_HASH: {"S": key.
|
|
343
|
+
Key={self.adapter.ER_KEY_HASH: {"S": key.state_path}},
|
|
334
344
|
UpdateExpression="set resource_status=:new_value",
|
|
335
345
|
ExpressionAttributeValues={":new_value": {"S": status.value}},
|
|
336
346
|
ReturnValues="UPDATED_NEW",
|
|
@@ -58,7 +58,7 @@ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
|
|
|
58
58
|
"""Return the desired state for early exit."""
|
|
59
59
|
return {
|
|
60
60
|
"version": QONTRACT_INTEGRATION_VERSION,
|
|
61
|
-
"specs": {spec.name: spec.
|
|
61
|
+
"specs": {spec.name: spec.model_dump() for spec in get_fleet_label_specs()},
|
|
62
62
|
}
|
|
63
63
|
|
|
64
64
|
def run(self, dry_run: bool) -> None:
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import json
|
|
2
1
|
import logging
|
|
3
2
|
import sys
|
|
4
3
|
from collections.abc import (
|
|
@@ -17,9 +16,11 @@ from reconcile import queries
|
|
|
17
16
|
from reconcile.status import ExitCodes
|
|
18
17
|
from reconcile.utils.aggregated_list import RunnerError
|
|
19
18
|
from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
|
|
19
|
+
from reconcile.utils.datetime_util import ensure_utc, utc_now
|
|
20
20
|
from reconcile.utils.defer import defer
|
|
21
21
|
from reconcile.utils.disabled_integrations import integration_is_enabled
|
|
22
22
|
from reconcile.utils.external_resources import get_external_resource_specs
|
|
23
|
+
from reconcile.utils.json import json_dumps
|
|
23
24
|
from reconcile.utils.openshift_resource import (
|
|
24
25
|
OpenshiftResource,
|
|
25
26
|
ResourceInventory,
|
|
@@ -39,12 +40,12 @@ def construct_gabi_oc_resource(
|
|
|
39
40
|
"kind": "ConfigMap",
|
|
40
41
|
"metadata": {"name": name, "annotations": {"qontract.recycle": "true"}},
|
|
41
42
|
"data": {
|
|
42
|
-
"config.json":
|
|
43
|
+
"config.json": json_dumps(
|
|
43
44
|
{
|
|
44
45
|
"expiration": str(expiration_date),
|
|
45
46
|
"users": users,
|
|
46
47
|
},
|
|
47
|
-
|
|
48
|
+
compact=True,
|
|
48
49
|
),
|
|
49
50
|
},
|
|
50
51
|
}
|
|
@@ -65,8 +66,10 @@ def fetch_desired_state(
|
|
|
65
66
|
gabi_instances: Iterable[Mapping], ri: ResourceInventory
|
|
66
67
|
) -> None:
|
|
67
68
|
for g in gabi_instances:
|
|
68
|
-
expiration_date =
|
|
69
|
-
|
|
69
|
+
expiration_date = ensure_utc(
|
|
70
|
+
datetime.strptime(g["expirationDate"], "%Y-%m-%d") # noqa: DTZ007
|
|
71
|
+
).date()
|
|
72
|
+
if (expiration_date - utc_now().date()).days > EXPIRATION_DAYS_MAX:
|
|
70
73
|
raise RunnerError(
|
|
71
74
|
f"The maximum expiration date of {g['name']} shall not "
|
|
72
75
|
f"exceed {EXPIRATION_DAYS_MAX} days from today"
|
reconcile/gcp_image_mirror.py
CHANGED
|
@@ -132,7 +132,7 @@ class QuayMirror:
|
|
|
132
132
|
mirror_creds = None
|
|
133
133
|
pull_credentials = item.mirror.pull_credentials
|
|
134
134
|
if pull_credentials:
|
|
135
|
-
raw_data = self.secret_reader.read_all(pull_credentials.
|
|
135
|
+
raw_data = self.secret_reader.read_all(pull_credentials.model_dump())
|
|
136
136
|
username = raw_data["user"]
|
|
137
137
|
password = raw_data["token"]
|
|
138
138
|
mirror_creds = f"{username}:{password}"
|
|
@@ -226,7 +226,7 @@ class QuayMirror:
|
|
|
226
226
|
return False
|
|
227
227
|
|
|
228
228
|
def _decode_push_secret(self, secret: VaultSecret) -> str:
|
|
229
|
-
raw_data = self.secret_reader.read_all(secret.
|
|
229
|
+
raw_data = self.secret_reader.read_all(secret.model_dump())
|
|
230
230
|
token = base64.b64decode(raw_data["token"]).decode()
|
|
231
231
|
return f"{raw_data['user']}:{token}"
|
|
232
232
|
|
reconcile/github_org.py
CHANGED
reconcile/github_owners.py
CHANGED
|
@@ -2,6 +2,7 @@ import logging
|
|
|
2
2
|
import os
|
|
3
3
|
|
|
4
4
|
import github
|
|
5
|
+
import github.NamedUser
|
|
5
6
|
from github import Github
|
|
6
7
|
from sretoolbox.utils import retry
|
|
7
8
|
|
|
@@ -100,4 +101,7 @@ def run(dry_run: bool) -> None:
|
|
|
100
101
|
|
|
101
102
|
if not dry_run:
|
|
102
103
|
gh_user = gh.get_user(github_username)
|
|
104
|
+
assert isinstance(
|
|
105
|
+
gh_user, github.NamedUser.NamedUser
|
|
106
|
+
) # make mypy happy
|
|
103
107
|
gh_org.add_to_members(gh_user, "admin")
|
reconcile/gitlab_housekeeping.py
CHANGED
|
@@ -6,7 +6,6 @@ from collections.abc import (
|
|
|
6
6
|
from contextlib import suppress
|
|
7
7
|
from dataclasses import dataclass
|
|
8
8
|
from datetime import (
|
|
9
|
-
UTC,
|
|
10
9
|
datetime,
|
|
11
10
|
timedelta,
|
|
12
11
|
)
|
|
@@ -30,6 +29,7 @@ from sretoolbox.utils import retry
|
|
|
30
29
|
|
|
31
30
|
from reconcile import queries
|
|
32
31
|
from reconcile.change_owners.change_types import ChangeTypePriority
|
|
32
|
+
from reconcile.utils.datetime_util import ensure_utc, from_utc_iso_format, utc_now
|
|
33
33
|
from reconcile.utils.gitlab_api import (
|
|
34
34
|
GitLabApi,
|
|
35
35
|
MRState,
|
|
@@ -72,7 +72,6 @@ HOLD_LABELS = [
|
|
|
72
72
|
]
|
|
73
73
|
|
|
74
74
|
QONTRACT_INTEGRATION = "gitlab-housekeeping"
|
|
75
|
-
DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
|
|
76
75
|
EXPIRATION_DATE_FORMAT = "%Y-%m-%d"
|
|
77
76
|
SQUASH_OPTION_ALWAYS = "always"
|
|
78
77
|
|
|
@@ -128,9 +127,7 @@ def _calculate_time_since_approval(approved_at: str) -> float:
|
|
|
128
127
|
Returns the number of minutes since a MR has been approved.
|
|
129
128
|
:param approved_at: the datetime the MR was approved in format %Y-%m-%dT%H:%M:%S.%fZ
|
|
130
129
|
"""
|
|
131
|
-
time_since_approval =
|
|
132
|
-
approved_at, DATE_FORMAT
|
|
133
|
-
)
|
|
130
|
+
time_since_approval = utc_now() - from_utc_iso_format(approved_at)
|
|
134
131
|
return time_since_approval.total_seconds() / 60
|
|
135
132
|
|
|
136
133
|
|
|
@@ -138,7 +135,7 @@ def get_timed_out_pipelines(
|
|
|
138
135
|
pipelines: list[ProjectMergeRequestPipeline],
|
|
139
136
|
pipeline_timeout: int = 60,
|
|
140
137
|
) -> list[ProjectMergeRequestPipeline]:
|
|
141
|
-
now =
|
|
138
|
+
now = utc_now()
|
|
142
139
|
|
|
143
140
|
pending_pipelines = [
|
|
144
141
|
p
|
|
@@ -152,7 +149,7 @@ def get_timed_out_pipelines(
|
|
|
152
149
|
timed_out_pipelines = []
|
|
153
150
|
|
|
154
151
|
for p in pending_pipelines:
|
|
155
|
-
update_time =
|
|
152
|
+
update_time = from_utc_iso_format(p.updated_at)
|
|
156
153
|
|
|
157
154
|
elapsed = (now - update_time).total_seconds()
|
|
158
155
|
|
|
@@ -279,7 +276,7 @@ def handle_stale_items(
|
|
|
279
276
|
) -> None:
|
|
280
277
|
LABEL = "stale" # noqa: N806
|
|
281
278
|
|
|
282
|
-
now =
|
|
279
|
+
now = utc_now()
|
|
283
280
|
for item in items:
|
|
284
281
|
if AUTO_MERGE in item.labels:
|
|
285
282
|
if item.merge_status == MRStatus.UNCHECKED:
|
|
@@ -287,7 +284,7 @@ def handle_stale_items(
|
|
|
287
284
|
item = gl.get_merge_request(item.iid)
|
|
288
285
|
if item.merge_status == MRStatus.CANNOT_BE_MERGED:
|
|
289
286
|
close_item(dry_run, gl, enable_closing, item_type, item)
|
|
290
|
-
update_date =
|
|
287
|
+
update_date = from_utc_iso_format(item.updated_at)
|
|
291
288
|
|
|
292
289
|
# if item is over days_interval
|
|
293
290
|
current_interval = now.date() - update_date.date()
|
|
@@ -315,10 +312,9 @@ def handle_stale_items(
|
|
|
315
312
|
if not cancel_notes:
|
|
316
313
|
continue
|
|
317
314
|
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
latest_cancel_note_date = max(d for d in cancel_notes_dates)
|
|
315
|
+
latest_cancel_note_date = max(
|
|
316
|
+
from_utc_iso_format(note.updated_at) for note in cancel_notes
|
|
317
|
+
)
|
|
322
318
|
# if the latest cancel note is under
|
|
323
319
|
# days_interval - remove 'stale' label
|
|
324
320
|
current_interval = now.date() - latest_cancel_note_date.date()
|
|
@@ -653,8 +649,10 @@ def publish_access_token_expiration_metrics(gl: GitLabApi) -> None:
|
|
|
653
649
|
|
|
654
650
|
for pat in pats:
|
|
655
651
|
if pat.active:
|
|
656
|
-
expiration_date =
|
|
657
|
-
|
|
652
|
+
expiration_date = ensure_utc(
|
|
653
|
+
datetime.strptime(pat.expires_at, EXPIRATION_DATE_FORMAT) # noqa: DTZ007
|
|
654
|
+
)
|
|
655
|
+
days_until_expiration = expiration_date.date() - utc_now().date()
|
|
658
656
|
gitlab_token_expiration.labels(pat.name).set(days_until_expiration.days)
|
|
659
657
|
else:
|
|
660
658
|
with suppress(KeyError, ValueError):
|
reconcile/gitlab_members.py
CHANGED
|
@@ -44,19 +44,13 @@ class GitlabUser(BaseModel):
|
|
|
44
44
|
access_level: int
|
|
45
45
|
|
|
46
46
|
|
|
47
|
-
class CurrentStateSpec(BaseModel):
|
|
47
|
+
class CurrentStateSpec(BaseModel, arbitrary_types_allowed=True):
|
|
48
48
|
members: dict[str, GroupMember]
|
|
49
49
|
|
|
50
|
-
class Config:
|
|
51
|
-
arbitrary_types_allowed = True
|
|
52
50
|
|
|
53
|
-
|
|
54
|
-
class DesiredStateSpec(BaseModel):
|
|
51
|
+
class DesiredStateSpec(BaseModel, arbitrary_types_allowed=True):
|
|
55
52
|
members: dict[str, GitlabUser]
|
|
56
53
|
|
|
57
|
-
class Config:
|
|
58
|
-
arbitrary_types_allowed = True
|
|
59
|
-
|
|
60
54
|
|
|
61
55
|
CurrentState = dict[str, CurrentStateSpec]
|
|
62
56
|
DesiredState = dict[str, DesiredStateSpec]
|
|
@@ -122,8 +116,8 @@ def build_desired_state_spec(
|
|
|
122
116
|
pagerduty_map,
|
|
123
117
|
get_username_method=lambda u: u.org_username,
|
|
124
118
|
)
|
|
125
|
-
for
|
|
126
|
-
gu = GitlabUser(user=
|
|
119
|
+
for pu in usernames_from_pagerduty:
|
|
120
|
+
gu = GitlabUser(user=pu, access_level=p_access_level)
|
|
127
121
|
add_or_update_user(desired_state_spec, gu)
|
|
128
122
|
return desired_state_spec
|
|
129
123
|
|
|
@@ -239,6 +233,6 @@ def reconcile_gitlab_members(
|
|
|
239
233
|
def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:
|
|
240
234
|
gqlapi = gql.get_api()
|
|
241
235
|
return {
|
|
242
|
-
"instance": get_gitlab_instance(gqlapi.query).
|
|
243
|
-
"permissions": [p.
|
|
236
|
+
"instance": get_gitlab_instance(gqlapi.query).model_dump(),
|
|
237
|
+
"permissions": [p.model_dump() for p in get_permissions(gqlapi.query)],
|
|
244
238
|
}
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
SQS Consumer to create Gitlab merge requests.
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
|
-
import json
|
|
6
5
|
import logging
|
|
7
6
|
import sys
|
|
8
7
|
from collections.abc import Callable
|
|
@@ -11,6 +10,7 @@ from reconcile import queries
|
|
|
11
10
|
from reconcile.utils import mr
|
|
12
11
|
from reconcile.utils.defer import defer
|
|
13
12
|
from reconcile.utils.gitlab_api import GitLabApi
|
|
13
|
+
from reconcile.utils.json import json_dumps
|
|
14
14
|
from reconcile.utils.secret_reader import SecretReader
|
|
15
15
|
from reconcile.utils.sqs_gateway import SQSGateway
|
|
16
16
|
|
|
@@ -51,7 +51,7 @@ def run(dry_run: str, gitlab_project_id: str, defer: Callable | None = None) ->
|
|
|
51
51
|
for m in messages:
|
|
52
52
|
receipt_handle, body = m[0], m[1]
|
|
53
53
|
logging.info(
|
|
54
|
-
"received message %s with body %s", receipt_handle[:6],
|
|
54
|
+
"received message %s with body %s", receipt_handle[:6], json_dumps(body)
|
|
55
55
|
)
|
|
56
56
|
|
|
57
57
|
if not dry_run:
|
reconcile/gitlab_owners.py
CHANGED
|
@@ -2,12 +2,12 @@ import logging
|
|
|
2
2
|
from collections.abc import Callable, Mapping
|
|
3
3
|
from typing import Any
|
|
4
4
|
|
|
5
|
-
from dateutil import parser as dateparser
|
|
6
5
|
from gitlab.v4.objects import ProjectMergeRequest
|
|
7
6
|
from sretoolbox.utils import threaded
|
|
8
7
|
|
|
9
8
|
from reconcile import queries
|
|
10
9
|
from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
|
|
10
|
+
from reconcile.utils.datetime_util import from_utc_iso_format
|
|
11
11
|
from reconcile.utils.defer import defer
|
|
12
12
|
from reconcile.utils.gitlab_api import (
|
|
13
13
|
GitLabApi,
|
|
@@ -49,12 +49,14 @@ class MRApproval:
|
|
|
49
49
|
self.dry_run = dry_run
|
|
50
50
|
self.persistent_lgtm = persistent_lgtm
|
|
51
51
|
|
|
52
|
-
# Get the date of the most recent commit (top commit) in the MR
|
|
53
|
-
self.top_commit_created_at =
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
52
|
+
# Get the date of the most recent commit (top commit) in the MR
|
|
53
|
+
self.top_commit_created_at = next(
|
|
54
|
+
(
|
|
55
|
+
from_utc_iso_format(commit.created_at)
|
|
56
|
+
for commit in merge_request.commits()
|
|
57
|
+
),
|
|
58
|
+
None,
|
|
59
|
+
)
|
|
58
60
|
|
|
59
61
|
def get_change_owners_map(self) -> dict[str, dict[str, list[str]]]:
|
|
60
62
|
"""
|
|
@@ -95,9 +97,10 @@ class MRApproval:
|
|
|
95
97
|
|
|
96
98
|
# Only interested in comments created after the top commit
|
|
97
99
|
# creation time
|
|
98
|
-
comment_created_at =
|
|
100
|
+
comment_created_at = from_utc_iso_format(comment.created_at)
|
|
99
101
|
if (
|
|
100
|
-
|
|
102
|
+
self.top_commit_created_at is not None
|
|
103
|
+
and comment_created_at < self.top_commit_created_at
|
|
101
104
|
and not self.persistent_lgtm
|
|
102
105
|
):
|
|
103
106
|
continue
|
|
@@ -185,9 +188,10 @@ class MRApproval:
|
|
|
185
188
|
# If the comment was created before the last commit,
|
|
186
189
|
# it means we had a push after the comment. In this case,
|
|
187
190
|
# we delete the comment and move on.
|
|
188
|
-
comment_created_at =
|
|
191
|
+
comment_created_at = from_utc_iso_format(comment.created_at)
|
|
189
192
|
if (
|
|
190
|
-
|
|
193
|
+
self.top_commit_created_at is not None
|
|
194
|
+
and comment_created_at < self.top_commit_created_at
|
|
191
195
|
and comment.note is not None
|
|
192
196
|
):
|
|
193
197
|
# Deleting stale comments
|