qontract-reconcile 0.10.2.dev345__py3-none-any.whl → 0.10.2.dev408__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {qontract_reconcile-0.10.2.dev345.dist-info → qontract_reconcile-0.10.2.dev408.dist-info}/METADATA +11 -10
- {qontract_reconcile-0.10.2.dev345.dist-info → qontract_reconcile-0.10.2.dev408.dist-info}/RECORD +126 -120
- reconcile/aus/base.py +17 -14
- reconcile/automated_actions/config/integration.py +12 -0
- reconcile/aws_account_manager/integration.py +2 -2
- reconcile/aws_ami_cleanup/integration.py +6 -7
- reconcile/aws_ami_share.py +69 -62
- reconcile/aws_cloudwatch_log_retention/integration.py +155 -126
- reconcile/aws_ecr_image_pull_secrets.py +2 -2
- reconcile/aws_iam_keys.py +1 -0
- reconcile/aws_saml_idp/integration.py +7 -1
- reconcile/aws_saml_roles/integration.py +9 -3
- reconcile/change_owners/change_owners.py +1 -1
- reconcile/change_owners/diff.py +2 -4
- reconcile/checkpoint.py +11 -3
- reconcile/cli.py +33 -8
- reconcile/dashdotdb_dora.py +4 -11
- reconcile/database_access_manager.py +118 -111
- reconcile/endpoints_discovery/integration.py +4 -1
- reconcile/endpoints_discovery/merge_request_manager.py +9 -11
- reconcile/external_resources/factories.py +5 -12
- reconcile/external_resources/integration.py +1 -1
- reconcile/external_resources/manager.py +5 -3
- reconcile/external_resources/meta.py +0 -1
- reconcile/external_resources/model.py +10 -10
- reconcile/external_resources/reconciler.py +5 -2
- reconcile/external_resources/secrets_sync.py +4 -6
- reconcile/external_resources/state.py +5 -4
- reconcile/gabi_authorized_users.py +8 -5
- reconcile/gitlab_housekeeping.py +13 -15
- reconcile/gitlab_mr_sqs_consumer.py +2 -2
- reconcile/gitlab_owners.py +15 -11
- reconcile/gql_definitions/automated_actions/instance.py +41 -2
- reconcile/gql_definitions/aws_ami_cleanup/aws_accounts.py +10 -0
- reconcile/gql_definitions/aws_cloudwatch_log_retention/aws_accounts.py +22 -61
- reconcile/gql_definitions/aws_saml_idp/aws_accounts.py +10 -0
- reconcile/gql_definitions/aws_saml_roles/aws_accounts.py +10 -0
- reconcile/gql_definitions/common/aws_vpc_requests.py +10 -0
- reconcile/gql_definitions/common/clusters.py +2 -0
- reconcile/gql_definitions/external_resources/external_resources_namespaces.py +84 -1
- reconcile/gql_definitions/external_resources/external_resources_settings.py +2 -0
- reconcile/gql_definitions/fragments/aws_account_common.py +2 -0
- reconcile/gql_definitions/fragments/aws_organization.py +33 -0
- reconcile/gql_definitions/fragments/aws_vpc_request.py +2 -0
- reconcile/gql_definitions/introspection.json +3474 -1986
- reconcile/gql_definitions/jira_permissions_validator/jira_boards_for_permissions_validator.py +4 -0
- reconcile/gql_definitions/terraform_init/aws_accounts.py +14 -0
- reconcile/gql_definitions/terraform_resources/terraform_resources_namespaces.py +33 -1
- reconcile/gql_definitions/terraform_tgw_attachments/aws_accounts.py +10 -0
- reconcile/jenkins_worker_fleets.py +1 -0
- reconcile/jira_permissions_validator.py +236 -121
- reconcile/ocm/types.py +6 -0
- reconcile/openshift_base.py +47 -1
- reconcile/openshift_cluster_bots.py +2 -1
- reconcile/openshift_resources_base.py +6 -2
- reconcile/openshift_saas_deploy.py +2 -2
- reconcile/openshift_saas_deploy_trigger_cleaner.py +3 -5
- reconcile/openshift_upgrade_watcher.py +3 -3
- reconcile/queries.py +131 -0
- reconcile/saas_auto_promotions_manager/subscriber.py +4 -3
- reconcile/slack_usergroups.py +4 -3
- reconcile/sql_query.py +1 -0
- reconcile/statuspage/integrations/maintenances.py +4 -3
- reconcile/statuspage/status.py +5 -8
- reconcile/templates/rosa-classic-cluster-creation.sh.j2 +4 -0
- reconcile/templates/rosa-hcp-cluster-creation.sh.j2 +3 -0
- reconcile/templating/renderer.py +2 -1
- reconcile/terraform_aws_route53.py +7 -1
- reconcile/terraform_init/integration.py +185 -21
- reconcile/terraform_resources.py +11 -1
- reconcile/terraform_tgw_attachments.py +7 -1
- reconcile/terraform_users.py +7 -0
- reconcile/terraform_vpc_peerings.py +14 -3
- reconcile/terraform_vpc_resources/integration.py +7 -0
- reconcile/typed_queries/aws_account_tags.py +41 -0
- reconcile/typed_queries/saas_files.py +2 -2
- reconcile/utils/aggregated_list.py +4 -3
- reconcile/utils/aws_api.py +51 -20
- reconcile/utils/aws_api_typed/api.py +38 -9
- reconcile/utils/aws_api_typed/cloudformation.py +149 -0
- reconcile/utils/aws_api_typed/logs.py +73 -0
- reconcile/utils/datetime_util.py +67 -0
- reconcile/utils/differ.py +2 -3
- reconcile/utils/early_exit_cache.py +3 -2
- reconcile/utils/expiration.py +7 -3
- reconcile/utils/external_resource_spec.py +24 -1
- reconcile/utils/filtering.py +1 -1
- reconcile/utils/helm.py +2 -1
- reconcile/utils/helpers.py +1 -1
- reconcile/utils/jinja2/utils.py +4 -96
- reconcile/utils/jira_client.py +82 -63
- reconcile/utils/jjb_client.py +9 -12
- reconcile/utils/jobcontroller/controller.py +1 -1
- reconcile/utils/jobcontroller/models.py +17 -1
- reconcile/utils/json.py +32 -0
- reconcile/utils/merge_request_manager/merge_request_manager.py +3 -3
- reconcile/utils/merge_request_manager/parser.py +2 -2
- reconcile/utils/mr/app_interface_reporter.py +2 -2
- reconcile/utils/mr/base.py +2 -2
- reconcile/utils/mr/notificator.py +2 -2
- reconcile/utils/mr/update_access_report_base.py +3 -4
- reconcile/utils/oc.py +113 -95
- reconcile/utils/oc_filters.py +3 -3
- reconcile/utils/ocm/products.py +6 -0
- reconcile/utils/ocm/search_filters.py +3 -6
- reconcile/utils/ocm/service_log.py +3 -5
- reconcile/utils/openshift_resource.py +10 -5
- reconcile/utils/output.py +3 -2
- reconcile/utils/pagerduty_api.py +5 -5
- reconcile/utils/runtime/integration.py +1 -2
- reconcile/utils/runtime/runner.py +2 -2
- reconcile/utils/saasherder/models.py +2 -1
- reconcile/utils/saasherder/saasherder.py +9 -7
- reconcile/utils/slack_api.py +24 -2
- reconcile/utils/sloth.py +171 -2
- reconcile/utils/sqs_gateway.py +2 -1
- reconcile/utils/state.py +2 -1
- reconcile/utils/terraform_client.py +4 -3
- reconcile/utils/terrascript_aws_client.py +165 -111
- reconcile/utils/vault.py +1 -1
- reconcile/vault_replication.py +107 -42
- tools/app_interface_reporter.py +4 -4
- tools/cli_commands/systems_and_tools.py +5 -1
- tools/qontract_cli.py +25 -13
- {qontract_reconcile-0.10.2.dev345.dist-info → qontract_reconcile-0.10.2.dev408.dist-info}/WHEEL +0 -0
- {qontract_reconcile-0.10.2.dev345.dist-info → qontract_reconcile-0.10.2.dev408.dist-info}/entry_points.txt +0 -0
|
@@ -2,31 +2,41 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
4
|
import re
|
|
5
|
-
import typing
|
|
6
5
|
from collections import defaultdict
|
|
7
6
|
from datetime import UTC, datetime, timedelta
|
|
8
|
-
from enum import Enum
|
|
9
7
|
from typing import TYPE_CHECKING
|
|
10
8
|
|
|
11
|
-
from botocore.exceptions import ClientError
|
|
12
9
|
from pydantic import BaseModel
|
|
13
10
|
|
|
14
|
-
from reconcile import queries
|
|
15
11
|
from reconcile.gql_definitions.aws_cloudwatch_log_retention.aws_accounts import (
|
|
16
12
|
AWSAccountCleanupOptionCloudWatchV1,
|
|
17
13
|
AWSAccountV1,
|
|
18
14
|
)
|
|
15
|
+
from reconcile.typed_queries.app_interface_vault_settings import (
|
|
16
|
+
get_app_interface_vault_settings,
|
|
17
|
+
)
|
|
18
|
+
from reconcile.typed_queries.aws_account_tags import get_aws_account_tags
|
|
19
19
|
from reconcile.typed_queries.aws_cloudwatch_log_retention.aws_accounts import (
|
|
20
20
|
get_aws_accounts,
|
|
21
21
|
)
|
|
22
|
+
from reconcile.typed_queries.external_resources import get_settings
|
|
22
23
|
from reconcile.utils import gql
|
|
23
|
-
from reconcile.utils.
|
|
24
|
+
from reconcile.utils.aws_api_typed.api import AWSApi, AWSStaticCredentials
|
|
25
|
+
from reconcile.utils.datetime_util import utc_now
|
|
26
|
+
from reconcile.utils.differ import diff_mappings
|
|
27
|
+
from reconcile.utils.secret_reader import create_secret_reader
|
|
28
|
+
from reconcile.utils.state import init_state
|
|
29
|
+
|
|
30
|
+
TAGS_KEY = "tags.json"
|
|
24
31
|
|
|
25
32
|
if TYPE_CHECKING:
|
|
26
33
|
from collections.abc import Iterable
|
|
27
34
|
|
|
28
35
|
from mypy_boto3_logs.type_defs import LogGroupTypeDef
|
|
29
36
|
|
|
37
|
+
from reconcile.utils.aws_api_typed.logs import AWSApiLogs
|
|
38
|
+
from reconcile.utils.gql import GqlApi
|
|
39
|
+
|
|
30
40
|
|
|
31
41
|
QONTRACT_INTEGRATION = "aws_cloudwatch_log_retention"
|
|
32
42
|
MANAGED_BY_INTEGRATION_KEY = "managed_by_integration"
|
|
@@ -35,7 +45,7 @@ DEFAULT_RETENTION_IN_DAYS = 90
|
|
|
35
45
|
|
|
36
46
|
|
|
37
47
|
class AWSCloudwatchCleanupOption(BaseModel):
|
|
38
|
-
regex:
|
|
48
|
+
regex: re.Pattern
|
|
39
49
|
retention_in_days: int
|
|
40
50
|
delete_empty_log_group: bool
|
|
41
51
|
|
|
@@ -67,16 +77,6 @@ def get_desired_cleanup_options_by_region(
|
|
|
67
77
|
return result
|
|
68
78
|
|
|
69
79
|
|
|
70
|
-
def create_awsapi_client(accounts: list[AWSAccountV1], thread_pool_size: int) -> AWSApi:
|
|
71
|
-
settings = queries.get_secret_reader_settings()
|
|
72
|
-
return AWSApi(
|
|
73
|
-
thread_pool_size,
|
|
74
|
-
[account.dict(by_alias=True) for account in accounts],
|
|
75
|
-
settings=settings,
|
|
76
|
-
init_users=False,
|
|
77
|
-
)
|
|
78
|
-
|
|
79
|
-
|
|
80
80
|
def is_empty(log_group: LogGroupTypeDef) -> bool:
|
|
81
81
|
return log_group["storedBytes"] == 0
|
|
82
82
|
|
|
@@ -85,47 +85,32 @@ def is_longer_than_retention(
|
|
|
85
85
|
log_group: LogGroupTypeDef,
|
|
86
86
|
desired_retention_days: int,
|
|
87
87
|
) -> bool:
|
|
88
|
-
return
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
class TagStatus(Enum):
|
|
94
|
-
NOT_SET = "NOT_SET"
|
|
95
|
-
MANAGED_BY_CURRENT_INTEGRATION = "MANAGED_BY_CURRENT_INTEGRATION"
|
|
96
|
-
MANAGED_BY_OTHER_INTEGRATION = "MANAGED_BY_OTHER_INTEGRATION"
|
|
88
|
+
return (
|
|
89
|
+
datetime.fromtimestamp(log_group["creationTime"] / 1000, tz=UTC)
|
|
90
|
+
+ timedelta(days=desired_retention_days)
|
|
91
|
+
< utc_now()
|
|
92
|
+
)
|
|
97
93
|
|
|
98
94
|
|
|
99
|
-
def
|
|
100
|
-
log_group: LogGroupTypeDef,
|
|
101
|
-
account_name: str,
|
|
102
|
-
region: str,
|
|
103
|
-
aws_api: AWSApi,
|
|
104
|
-
) -> TagStatus:
|
|
105
|
-
tags = aws_api.get_cloudwatch_log_group_tags(
|
|
106
|
-
account_name,
|
|
107
|
-
log_group["arn"],
|
|
108
|
-
region,
|
|
109
|
-
)
|
|
95
|
+
def _is_managed_by_other_integration(tags: dict[str, str]) -> bool:
|
|
110
96
|
managed_by_integration = tags.get(MANAGED_BY_INTEGRATION_KEY)
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
return TagStatus.MANAGED_BY_OTHER_INTEGRATION
|
|
97
|
+
return (
|
|
98
|
+
managed_by_integration is not None
|
|
99
|
+
and managed_by_integration != QONTRACT_INTEGRATION
|
|
100
|
+
)
|
|
116
101
|
|
|
117
102
|
|
|
118
103
|
def _reconcile_log_group(
|
|
119
104
|
dry_run: bool,
|
|
120
|
-
|
|
105
|
+
log_group: LogGroupTypeDef,
|
|
121
106
|
desired_cleanup_options: Iterable[AWSCloudwatchCleanupOption],
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
107
|
+
desired_tags: dict[str, str],
|
|
108
|
+
last_tags: dict[str, str],
|
|
109
|
+
aws_api_logs: AWSApiLogs,
|
|
125
110
|
) -> None:
|
|
126
|
-
current_retention_in_days =
|
|
127
|
-
log_group_name =
|
|
128
|
-
log_group_arn =
|
|
111
|
+
current_retention_in_days = log_group.get("retentionInDays")
|
|
112
|
+
log_group_name = log_group["logGroupName"]
|
|
113
|
+
log_group_arn = log_group["arn"]
|
|
129
114
|
|
|
130
115
|
desired_cleanup_option = _find_desired_cleanup_option(
|
|
131
116
|
log_group_name, desired_cleanup_options
|
|
@@ -133,54 +118,66 @@ def _reconcile_log_group(
|
|
|
133
118
|
|
|
134
119
|
if (
|
|
135
120
|
desired_cleanup_option.delete_empty_log_group
|
|
136
|
-
and is_empty(
|
|
121
|
+
and is_empty(log_group)
|
|
137
122
|
and is_longer_than_retention(
|
|
138
|
-
|
|
123
|
+
log_group, desired_cleanup_option.retention_in_days
|
|
139
124
|
)
|
|
140
125
|
):
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
!= TagStatus.MANAGED_BY_OTHER_INTEGRATION
|
|
144
|
-
):
|
|
126
|
+
tags = aws_api_logs.get_tags(log_group_arn)
|
|
127
|
+
if not _is_managed_by_other_integration(tags):
|
|
145
128
|
logging.info(
|
|
146
129
|
"Deleting empty log group %s",
|
|
147
130
|
log_group_arn,
|
|
148
131
|
)
|
|
149
132
|
if not dry_run:
|
|
150
|
-
|
|
133
|
+
aws_api_logs.delete_log_group(log_group_name)
|
|
151
134
|
return
|
|
152
135
|
|
|
153
|
-
if
|
|
136
|
+
if (
|
|
137
|
+
current_retention_in_days == desired_cleanup_option.retention_in_days
|
|
138
|
+
and last_tags == desired_tags
|
|
139
|
+
):
|
|
154
140
|
return
|
|
155
141
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
142
|
+
current_tags = aws_api_logs.get_tags(log_group_arn)
|
|
143
|
+
if _is_managed_by_other_integration(current_tags):
|
|
144
|
+
return
|
|
145
|
+
|
|
146
|
+
diff_result = diff_mappings(
|
|
147
|
+
current=current_tags,
|
|
148
|
+
desired=desired_tags,
|
|
149
|
+
)
|
|
150
|
+
if to_delete := diff_result.delete.keys() & last_tags.keys():
|
|
151
|
+
logging.info(
|
|
152
|
+
"Deleting tags %s for log group %s",
|
|
153
|
+
to_delete,
|
|
154
|
+
log_group_arn,
|
|
155
|
+
)
|
|
156
|
+
if not dry_run:
|
|
157
|
+
aws_api_logs.delete_tags(
|
|
165
158
|
log_group_arn,
|
|
159
|
+
to_delete,
|
|
166
160
|
)
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
161
|
+
if diff_result.add or diff_result.change:
|
|
162
|
+
logging.info(
|
|
163
|
+
"Setting tags %s for log group %s",
|
|
164
|
+
desired_tags,
|
|
165
|
+
log_group_arn,
|
|
166
|
+
)
|
|
167
|
+
if not dry_run:
|
|
168
|
+
aws_api_logs.set_tags(log_group_arn, desired_tags)
|
|
171
169
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
)
|
|
177
|
-
if not dry_run:
|
|
178
|
-
awsapi.set_cloudwatch_log_retention(
|
|
179
|
-
account_name,
|
|
180
|
-
log_group_name,
|
|
170
|
+
if current_retention_in_days != desired_cleanup_option.retention_in_days:
|
|
171
|
+
logging.info(
|
|
172
|
+
"Setting %s retention days to %d",
|
|
173
|
+
log_group_arn,
|
|
181
174
|
desired_cleanup_option.retention_in_days,
|
|
182
|
-
region,
|
|
183
175
|
)
|
|
176
|
+
if not dry_run:
|
|
177
|
+
aws_api_logs.put_retention_policy(
|
|
178
|
+
log_group_name,
|
|
179
|
+
desired_cleanup_option.retention_in_days,
|
|
180
|
+
)
|
|
184
181
|
|
|
185
182
|
|
|
186
183
|
def _find_desired_cleanup_option(
|
|
@@ -191,60 +188,63 @@ def _find_desired_cleanup_option(
|
|
|
191
188
|
Find the first cleanup option that regex matches the log group name.
|
|
192
189
|
If no match is found, return the default cleanup option.
|
|
193
190
|
|
|
194
|
-
:
|
|
195
|
-
|
|
196
|
-
|
|
191
|
+
Args:
|
|
192
|
+
log_group_name: The name of the log group.
|
|
193
|
+
desired_cleanup_options: A list of desired cleanup options.
|
|
194
|
+
Returns:
|
|
195
|
+
The matching cleanup option or the default one.
|
|
197
196
|
"""
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
197
|
+
for option in desired_cleanup_options:
|
|
198
|
+
if option.regex.match(log_group_name):
|
|
199
|
+
return option
|
|
200
|
+
return DEFAULT_AWS_CLOUDWATCH_CLEANUP_OPTION
|
|
202
201
|
|
|
203
202
|
|
|
204
203
|
def _reconcile_log_groups(
|
|
205
204
|
dry_run: bool,
|
|
206
205
|
aws_account: AWSAccountV1,
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
206
|
+
last_tags: dict[str, str],
|
|
207
|
+
default_tags: dict[str, str],
|
|
208
|
+
automation_token: dict[str, str],
|
|
209
|
+
) -> dict[str, str]:
|
|
210
|
+
desired_tags = (
|
|
211
|
+
default_tags | get_aws_account_tags(aws_account.organization) | MANAGED_TAG
|
|
212
212
|
)
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
213
|
+
for (
|
|
214
|
+
region,
|
|
215
|
+
desired_cleanup_options,
|
|
216
|
+
) in get_desired_cleanup_options_by_region(aws_account).items():
|
|
217
|
+
aws_credentials = AWSStaticCredentials(
|
|
218
|
+
access_key_id=automation_token["aws_access_key_id"],
|
|
219
|
+
secret_access_key=automation_token["aws_secret_access_key"],
|
|
220
|
+
region=region,
|
|
221
|
+
)
|
|
222
|
+
with AWSApi(aws_credentials) as aws_api:
|
|
223
|
+
aws_api_logs = aws_api.logs
|
|
224
|
+
try:
|
|
225
|
+
for log_group in aws_api_logs.get_log_groups():
|
|
226
|
+
_reconcile_log_group(
|
|
227
|
+
dry_run=dry_run,
|
|
228
|
+
log_group=log_group,
|
|
229
|
+
desired_cleanup_options=desired_cleanup_options,
|
|
230
|
+
desired_tags=desired_tags,
|
|
231
|
+
last_tags=last_tags,
|
|
232
|
+
aws_api_logs=aws_api_logs,
|
|
233
|
+
)
|
|
234
|
+
except aws_api_logs.client.exceptions.ClientError as e:
|
|
235
|
+
logging.error(
|
|
236
|
+
"Error reconciling log groups for %s: %s",
|
|
237
|
+
aws_account.name,
|
|
238
|
+
e,
|
|
229
239
|
)
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
logging.info(
|
|
233
|
-
"Access denied for aws account %s. Skipping...",
|
|
234
|
-
account_name,
|
|
235
|
-
)
|
|
236
|
-
else:
|
|
237
|
-
logging.error(
|
|
238
|
-
"Error reconciling log groups for %s: %s",
|
|
239
|
-
account_name,
|
|
240
|
-
e,
|
|
241
|
-
)
|
|
240
|
+
return last_tags
|
|
241
|
+
return desired_tags
|
|
242
242
|
|
|
243
243
|
|
|
244
|
-
def get_active_aws_accounts() -> list[AWSAccountV1]:
|
|
244
|
+
def get_active_aws_accounts(gql_api: GqlApi) -> list[AWSAccountV1]:
|
|
245
245
|
return [
|
|
246
246
|
account
|
|
247
|
-
for account in get_aws_accounts(
|
|
247
|
+
for account in get_aws_accounts(gql_api)
|
|
248
248
|
if not (
|
|
249
249
|
account.disable
|
|
250
250
|
and account.disable.integrations
|
|
@@ -253,8 +253,37 @@ def get_active_aws_accounts() -> list[AWSAccountV1]:
|
|
|
253
253
|
]
|
|
254
254
|
|
|
255
255
|
|
|
256
|
-
def
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
256
|
+
def get_default_tags(gql_api: GqlApi) -> dict[str, str]:
|
|
257
|
+
try:
|
|
258
|
+
return get_settings(gql_api.query).default_tags
|
|
259
|
+
except ValueError:
|
|
260
|
+
# no settings found
|
|
261
|
+
return {}
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def run(dry_run: bool) -> None:
|
|
265
|
+
gql_api = gql.get_api()
|
|
266
|
+
aws_accounts = get_active_aws_accounts(gql_api)
|
|
267
|
+
vault_settings = get_app_interface_vault_settings(query_func=gql_api.query)
|
|
268
|
+
secret_reader = create_secret_reader(use_vault=vault_settings.vault)
|
|
269
|
+
default_tags = get_default_tags(gql_api)
|
|
270
|
+
|
|
271
|
+
with init_state(
|
|
272
|
+
integration=QONTRACT_INTEGRATION,
|
|
273
|
+
secret_reader=secret_reader,
|
|
274
|
+
) as state:
|
|
275
|
+
last_tags = state.get(TAGS_KEY, {})
|
|
276
|
+
desired_tags = {
|
|
277
|
+
aws_account.name: _reconcile_log_groups(
|
|
278
|
+
dry_run=dry_run,
|
|
279
|
+
aws_account=aws_account,
|
|
280
|
+
last_tags=last_tags.get(aws_account.name, {}),
|
|
281
|
+
default_tags=default_tags,
|
|
282
|
+
automation_token=secret_reader.read_all_secret(
|
|
283
|
+
aws_account.automation_token
|
|
284
|
+
),
|
|
285
|
+
)
|
|
286
|
+
for aws_account in aws_accounts
|
|
287
|
+
}
|
|
288
|
+
if not dry_run and desired_tags != last_tags:
|
|
289
|
+
state.add(TAGS_KEY, desired_tags, force=True)
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import base64
|
|
2
|
-
import json
|
|
3
2
|
import logging
|
|
4
3
|
from collections.abc import Mapping
|
|
5
4
|
from typing import Any
|
|
6
5
|
|
|
7
6
|
from reconcile import queries
|
|
8
7
|
from reconcile.utils.aws_api import AWSApi
|
|
8
|
+
from reconcile.utils.json import json_dumps
|
|
9
9
|
from reconcile.utils.vault import VaultClient
|
|
10
10
|
|
|
11
11
|
QONTRACT_INTEGRATION = "aws-ecr-image-pull-secrets"
|
|
@@ -35,7 +35,7 @@ def construct_dockercfg_secret_data(data: Mapping[str, Any]) -> dict[str, str]:
|
|
|
35
35
|
}
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
-
return {".dockerconfigjson": enc_dec(
|
|
38
|
+
return {".dockerconfigjson": enc_dec(json_dumps(data))}
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
def construct_basic_auth_secret_data(data: Mapping[str, Any]) -> dict[str, str]:
|
reconcile/aws_iam_keys.py
CHANGED
|
@@ -19,6 +19,7 @@ from reconcile.gql_definitions.aws_saml_idp.aws_accounts import (
|
|
|
19
19
|
query as aws_accounts_query,
|
|
20
20
|
)
|
|
21
21
|
from reconcile.status import ExitCodes
|
|
22
|
+
from reconcile.typed_queries.external_resources import get_settings
|
|
22
23
|
from reconcile.utils import gql
|
|
23
24
|
from reconcile.utils.aws_api import AWSApi
|
|
24
25
|
from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
|
|
@@ -125,13 +126,18 @@ class AwsSamlIdpIntegration(QontractReconcileIntegration[AwsSamlIdpIntegrationPa
|
|
|
125
126
|
gql_api.query, account_name=self.params.account_name
|
|
126
127
|
)
|
|
127
128
|
aws_accounts_dict = [account.dict(by_alias=True) for account in aws_accounts]
|
|
128
|
-
|
|
129
|
+
try:
|
|
130
|
+
default_tags = get_settings().default_tags
|
|
131
|
+
except ValueError:
|
|
132
|
+
# no external resources settings found
|
|
133
|
+
default_tags = None
|
|
129
134
|
ts = TerrascriptClient(
|
|
130
135
|
self.name.replace("-", "_"),
|
|
131
136
|
"",
|
|
132
137
|
self.params.thread_pool_size,
|
|
133
138
|
aws_accounts_dict,
|
|
134
139
|
secret_reader=self.secret_reader,
|
|
140
|
+
default_tags=default_tags,
|
|
135
141
|
)
|
|
136
142
|
|
|
137
143
|
for saml_idp_config in self.build_saml_idp_config(
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import json
|
|
2
1
|
import logging
|
|
3
2
|
import sys
|
|
4
3
|
from collections.abc import (
|
|
@@ -22,6 +21,7 @@ from reconcile.gql_definitions.aws_saml_roles.roles import (
|
|
|
22
21
|
query as roles_query,
|
|
23
22
|
)
|
|
24
23
|
from reconcile.status import ExitCodes
|
|
24
|
+
from reconcile.typed_queries.external_resources import get_settings
|
|
25
25
|
from reconcile.utils import gql
|
|
26
26
|
from reconcile.utils.aws_api import AWSApi
|
|
27
27
|
from reconcile.utils.aws_helper import unique_sso_aws_accounts
|
|
@@ -32,6 +32,7 @@ from reconcile.utils.extended_early_exit import (
|
|
|
32
32
|
ExtendedEarlyExitRunnerResult,
|
|
33
33
|
extended_early_exit_run,
|
|
34
34
|
)
|
|
35
|
+
from reconcile.utils.json import json_dumps
|
|
35
36
|
from reconcile.utils.runtime.integration import (
|
|
36
37
|
PydanticRunParams,
|
|
37
38
|
QontractReconcileIntegration,
|
|
@@ -87,7 +88,7 @@ class CustomPolicy(BaseModel):
|
|
|
87
88
|
|
|
88
89
|
See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html
|
|
89
90
|
"""
|
|
90
|
-
if len(
|
|
91
|
+
if len(json_dumps(v, compact=True)) > 6144:
|
|
91
92
|
raise ValueError(
|
|
92
93
|
f"The policy document '{v}' is too large. AWS policy documents must be 6144 characters or less (w/o white spaces)."
|
|
93
94
|
)
|
|
@@ -253,13 +254,18 @@ class AwsSamlRolesIntegration(
|
|
|
253
254
|
)
|
|
254
255
|
aws_accounts_dict = [account.dict(by_alias=True) for account in aws_accounts]
|
|
255
256
|
aws_roles = self.get_roles(gql_api.query, account_name=self.params.account_name)
|
|
256
|
-
|
|
257
|
+
try:
|
|
258
|
+
default_tags = get_settings().default_tags
|
|
259
|
+
except ValueError:
|
|
260
|
+
# no external resources settings found
|
|
261
|
+
default_tags = None
|
|
257
262
|
ts = TerrascriptClient(
|
|
258
263
|
self.name.replace("-", "_"),
|
|
259
264
|
"",
|
|
260
265
|
self.params.thread_pool_size,
|
|
261
266
|
aws_accounts_dict,
|
|
262
267
|
secret_reader=self.secret_reader,
|
|
268
|
+
default_tags=default_tags,
|
|
263
269
|
)
|
|
264
270
|
self.populate_saml_iam_roles(ts, aws_roles)
|
|
265
271
|
working_dirs = ts.dump(print_to_file=self.params.print_to_file)
|
|
@@ -140,7 +140,7 @@ def write_coverage_report_to_mr(
|
|
|
140
140
|
approver_reachability = set()
|
|
141
141
|
for d in change_decisions:
|
|
142
142
|
approvers = [
|
|
143
|
-
f"{cr.context} - {' '.join([f'@{a.org_username}' if a.tag_on_merge_requests else a.org_username for a in cr.approvers])}"
|
|
143
|
+
f"{cr.context} - {' '.join([f'@{a.org_username}' if (a.tag_on_merge_requests or len(cr.approvers) == 1) else a.org_username for a in cr.approvers])}"
|
|
144
144
|
for cr in d.change_responsibles
|
|
145
145
|
]
|
|
146
146
|
if d.coverable_by_fragment_decisions:
|
reconcile/change_owners/diff.py
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import copy
|
|
2
|
-
import json
|
|
3
2
|
from dataclasses import dataclass
|
|
4
3
|
from enum import Enum
|
|
5
4
|
from functools import reduce
|
|
@@ -11,6 +10,7 @@ from deepdiff.helper import CannotCompare
|
|
|
11
10
|
from deepdiff.model import DiffLevel
|
|
12
11
|
from deepdiff.path import parse_path
|
|
13
12
|
|
|
13
|
+
from reconcile.utils.json import json_dumps
|
|
14
14
|
from reconcile.utils.jsonpath import parse_jsonpath
|
|
15
15
|
|
|
16
16
|
|
|
@@ -75,7 +75,7 @@ class Diff:
|
|
|
75
75
|
def _value_repr(self, value: Any | None) -> str | None:
|
|
76
76
|
if value:
|
|
77
77
|
if isinstance(value, dict | list):
|
|
78
|
-
return
|
|
78
|
+
return json_dumps(value, indent=2)
|
|
79
79
|
return str(value)
|
|
80
80
|
return value
|
|
81
81
|
|
|
@@ -251,8 +251,6 @@ def deepdiff_path_to_jsonpath(deep_diff_path: str) -> jsonpath_ng.JSONPath:
|
|
|
251
251
|
case int():
|
|
252
252
|
return jsonpath_ng.Index(element)
|
|
253
253
|
case str():
|
|
254
|
-
if "." in element:
|
|
255
|
-
return jsonpath_ng.Fields(f"'{element}'")
|
|
256
254
|
return jsonpath_ng.Fields(element)
|
|
257
255
|
|
|
258
256
|
path_parts = [build_jsonpath_part(p) for p in parse_path(deep_diff_path)]
|
reconcile/checkpoint.py
CHANGED
|
@@ -26,6 +26,7 @@ from jira import Issue
|
|
|
26
26
|
|
|
27
27
|
from reconcile.utils.constants import PROJ_ROOT
|
|
28
28
|
from reconcile.utils.jira_client import JiraClient
|
|
29
|
+
from reconcile.utils.secret_reader import SecretReaderBase
|
|
29
30
|
|
|
30
31
|
DEFAULT_CHECKPOINT_LABELS = ("sre-checkpoint",)
|
|
31
32
|
|
|
@@ -118,8 +119,8 @@ def file_ticket(
|
|
|
118
119
|
def report_invalid_metadata(
|
|
119
120
|
app: Mapping[str, Any],
|
|
120
121
|
path: str,
|
|
121
|
-
board: Mapping[str,
|
|
122
|
-
|
|
122
|
+
board: Mapping[str, Any],
|
|
123
|
+
secret_reader: SecretReaderBase,
|
|
123
124
|
parent: str,
|
|
124
125
|
dry_run: bool = False,
|
|
125
126
|
) -> None:
|
|
@@ -150,7 +151,14 @@ def report_invalid_metadata(
|
|
|
150
151
|
path=path,
|
|
151
152
|
)
|
|
152
153
|
else:
|
|
153
|
-
jira = JiraClient(
|
|
154
|
+
jira = JiraClient.create(
|
|
155
|
+
project_name=board["name"],
|
|
156
|
+
token=secret_reader.read_secret(board["server"]["token"]),
|
|
157
|
+
email=secret_reader.read_secret(board["server"]["email"])
|
|
158
|
+
if board["server"]["email"]
|
|
159
|
+
else None,
|
|
160
|
+
server_url=board["server"]["server_url"],
|
|
161
|
+
)
|
|
154
162
|
do_cut = partial(
|
|
155
163
|
file_ticket, # type: ignore
|
|
156
164
|
jira=jira,
|
reconcile/cli.py
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
# ruff: noqa: PLC0415 - `import` should be at the top-level of a file
|
|
2
2
|
import faulthandler
|
|
3
|
-
import json
|
|
4
3
|
import logging
|
|
5
4
|
import os
|
|
6
5
|
import re
|
|
@@ -31,6 +30,7 @@ from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
|
|
|
31
30
|
from reconcile.utils.exceptions import PrintToFileInGitRepositoryError
|
|
32
31
|
from reconcile.utils.git import is_file_in_git_repo
|
|
33
32
|
from reconcile.utils.gql import GqlApiSingleton
|
|
33
|
+
from reconcile.utils.json import json_dumps
|
|
34
34
|
from reconcile.utils.promtool import PROMTOOL_VERSION, PROMTOOL_VERSION_REGEX
|
|
35
35
|
from reconcile.utils.runtime.environment import init_env
|
|
36
36
|
from reconcile.utils.runtime.integration import (
|
|
@@ -608,7 +608,7 @@ def run_class_integration(
|
|
|
608
608
|
if dump_schemas_file:
|
|
609
609
|
gqlapi = gql.get_api()
|
|
610
610
|
with open(dump_schemas_file, "w", encoding="locale") as f:
|
|
611
|
-
f.write(
|
|
611
|
+
f.write(json_dumps(gqlapi.get_queried_schemas()))
|
|
612
612
|
|
|
613
613
|
|
|
614
614
|
@click.group()
|
|
@@ -1028,7 +1028,7 @@ def aws_account_manager(
|
|
|
1028
1028
|
"--state-tmpl-resource",
|
|
1029
1029
|
help="Resource name of the state template-collection template in the app-interface.",
|
|
1030
1030
|
required=True,
|
|
1031
|
-
default="/terraform-init/terraform-state.yml",
|
|
1031
|
+
default="/terraform-init/terraform-state.yml.j2",
|
|
1032
1032
|
)
|
|
1033
1033
|
@click.option(
|
|
1034
1034
|
"--template-collection-root-path",
|
|
@@ -1036,12 +1036,26 @@ def aws_account_manager(
|
|
|
1036
1036
|
required=True,
|
|
1037
1037
|
default="data/templating/collections/terraform-init",
|
|
1038
1038
|
)
|
|
1039
|
+
@click.option(
|
|
1040
|
+
"--cloudformation-template-resource",
|
|
1041
|
+
help="Resource name of the CloudFormation template to create the S3 bucket",
|
|
1042
|
+
required=True,
|
|
1043
|
+
default="/terraform-init/terraform-state-s3-bucket.yaml",
|
|
1044
|
+
)
|
|
1045
|
+
@click.option(
|
|
1046
|
+
"--cloudformation-import-template-resource",
|
|
1047
|
+
help="Resource name of the CloudFormation template to import existing S3 bucket",
|
|
1048
|
+
required=True,
|
|
1049
|
+
default="/terraform-init/terraform-state-s3-bucket-import.yaml",
|
|
1050
|
+
)
|
|
1039
1051
|
@click.pass_context
|
|
1040
1052
|
def terraform_init(
|
|
1041
1053
|
ctx: click.Context,
|
|
1042
1054
|
account_name: str | None,
|
|
1043
1055
|
state_tmpl_resource: str,
|
|
1044
1056
|
template_collection_root_path: str,
|
|
1057
|
+
cloudformation_template_resource: str,
|
|
1058
|
+
cloudformation_import_template_resource: str,
|
|
1045
1059
|
) -> None:
|
|
1046
1060
|
from reconcile.terraform_init.integration import (
|
|
1047
1061
|
TerraformInitIntegration,
|
|
@@ -1054,6 +1068,8 @@ def terraform_init(
|
|
|
1054
1068
|
account_name=account_name,
|
|
1055
1069
|
state_tmpl_resource=state_tmpl_resource,
|
|
1056
1070
|
template_collection_root_path=template_collection_root_path,
|
|
1071
|
+
cloudformation_template_resource=cloudformation_template_resource,
|
|
1072
|
+
cloudformation_import_template_resource=cloudformation_import_template_resource,
|
|
1057
1073
|
)
|
|
1058
1074
|
),
|
|
1059
1075
|
ctx=ctx,
|
|
@@ -1135,9 +1151,17 @@ def jenkins_webhooks_cleaner(ctx: click.Context) -> None:
|
|
|
1135
1151
|
"--jira-board-name", help="The Jira board to act on.", default=None, multiple=True
|
|
1136
1152
|
)
|
|
1137
1153
|
@click.option("--board-check-interval", help="Check interval in minutes", default=120)
|
|
1154
|
+
@click.option(
|
|
1155
|
+
"--use-cache/--no-use-cache",
|
|
1156
|
+
default=True,
|
|
1157
|
+
help="Use cached results for validation.",
|
|
1158
|
+
)
|
|
1138
1159
|
@click.pass_context
|
|
1139
1160
|
def jira_permissions_validator(
|
|
1140
|
-
ctx: click.Context,
|
|
1161
|
+
ctx: click.Context,
|
|
1162
|
+
jira_board_name: Iterable[str] | None,
|
|
1163
|
+
board_check_interval: int,
|
|
1164
|
+
use_cache: bool,
|
|
1141
1165
|
) -> None:
|
|
1142
1166
|
import reconcile.jira_permissions_validator
|
|
1143
1167
|
|
|
@@ -1146,6 +1170,7 @@ def jira_permissions_validator(
|
|
|
1146
1170
|
ctx,
|
|
1147
1171
|
jira_board_name=jira_board_name,
|
|
1148
1172
|
board_check_interval_sec=board_check_interval * 60,
|
|
1173
|
+
use_cache=use_cache,
|
|
1149
1174
|
)
|
|
1150
1175
|
|
|
1151
1176
|
|
|
@@ -1270,14 +1295,14 @@ def aws_ami_cleanup(ctx: click.Context, thread_pool_size: int) -> None:
|
|
|
1270
1295
|
run_integration(reconcile.aws_ami_cleanup.integration, ctx, thread_pool_size)
|
|
1271
1296
|
|
|
1272
1297
|
|
|
1273
|
-
@integration.command(short_help="Set up retention period for Cloudwatch logs.")
|
|
1274
|
-
@threaded()
|
|
1298
|
+
@integration.command(short_help="Set up retention period and tags for Cloudwatch logs.")
|
|
1275
1299
|
@click.pass_context
|
|
1276
|
-
def aws_cloudwatch_log_retention(ctx: click.Context
|
|
1300
|
+
def aws_cloudwatch_log_retention(ctx: click.Context) -> None:
|
|
1277
1301
|
import reconcile.aws_cloudwatch_log_retention.integration
|
|
1278
1302
|
|
|
1279
1303
|
run_integration(
|
|
1280
|
-
reconcile.aws_cloudwatch_log_retention.integration,
|
|
1304
|
+
reconcile.aws_cloudwatch_log_retention.integration,
|
|
1305
|
+
ctx,
|
|
1281
1306
|
)
|
|
1282
1307
|
|
|
1283
1308
|
|