qontract-reconcile 0.10.2.dev345__py3-none-any.whl → 0.10.2.dev408__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. {qontract_reconcile-0.10.2.dev345.dist-info → qontract_reconcile-0.10.2.dev408.dist-info}/METADATA +11 -10
  2. {qontract_reconcile-0.10.2.dev345.dist-info → qontract_reconcile-0.10.2.dev408.dist-info}/RECORD +126 -120
  3. reconcile/aus/base.py +17 -14
  4. reconcile/automated_actions/config/integration.py +12 -0
  5. reconcile/aws_account_manager/integration.py +2 -2
  6. reconcile/aws_ami_cleanup/integration.py +6 -7
  7. reconcile/aws_ami_share.py +69 -62
  8. reconcile/aws_cloudwatch_log_retention/integration.py +155 -126
  9. reconcile/aws_ecr_image_pull_secrets.py +2 -2
  10. reconcile/aws_iam_keys.py +1 -0
  11. reconcile/aws_saml_idp/integration.py +7 -1
  12. reconcile/aws_saml_roles/integration.py +9 -3
  13. reconcile/change_owners/change_owners.py +1 -1
  14. reconcile/change_owners/diff.py +2 -4
  15. reconcile/checkpoint.py +11 -3
  16. reconcile/cli.py +33 -8
  17. reconcile/dashdotdb_dora.py +4 -11
  18. reconcile/database_access_manager.py +118 -111
  19. reconcile/endpoints_discovery/integration.py +4 -1
  20. reconcile/endpoints_discovery/merge_request_manager.py +9 -11
  21. reconcile/external_resources/factories.py +5 -12
  22. reconcile/external_resources/integration.py +1 -1
  23. reconcile/external_resources/manager.py +5 -3
  24. reconcile/external_resources/meta.py +0 -1
  25. reconcile/external_resources/model.py +10 -10
  26. reconcile/external_resources/reconciler.py +5 -2
  27. reconcile/external_resources/secrets_sync.py +4 -6
  28. reconcile/external_resources/state.py +5 -4
  29. reconcile/gabi_authorized_users.py +8 -5
  30. reconcile/gitlab_housekeeping.py +13 -15
  31. reconcile/gitlab_mr_sqs_consumer.py +2 -2
  32. reconcile/gitlab_owners.py +15 -11
  33. reconcile/gql_definitions/automated_actions/instance.py +41 -2
  34. reconcile/gql_definitions/aws_ami_cleanup/aws_accounts.py +10 -0
  35. reconcile/gql_definitions/aws_cloudwatch_log_retention/aws_accounts.py +22 -61
  36. reconcile/gql_definitions/aws_saml_idp/aws_accounts.py +10 -0
  37. reconcile/gql_definitions/aws_saml_roles/aws_accounts.py +10 -0
  38. reconcile/gql_definitions/common/aws_vpc_requests.py +10 -0
  39. reconcile/gql_definitions/common/clusters.py +2 -0
  40. reconcile/gql_definitions/external_resources/external_resources_namespaces.py +84 -1
  41. reconcile/gql_definitions/external_resources/external_resources_settings.py +2 -0
  42. reconcile/gql_definitions/fragments/aws_account_common.py +2 -0
  43. reconcile/gql_definitions/fragments/aws_organization.py +33 -0
  44. reconcile/gql_definitions/fragments/aws_vpc_request.py +2 -0
  45. reconcile/gql_definitions/introspection.json +3474 -1986
  46. reconcile/gql_definitions/jira_permissions_validator/jira_boards_for_permissions_validator.py +4 -0
  47. reconcile/gql_definitions/terraform_init/aws_accounts.py +14 -0
  48. reconcile/gql_definitions/terraform_resources/terraform_resources_namespaces.py +33 -1
  49. reconcile/gql_definitions/terraform_tgw_attachments/aws_accounts.py +10 -0
  50. reconcile/jenkins_worker_fleets.py +1 -0
  51. reconcile/jira_permissions_validator.py +236 -121
  52. reconcile/ocm/types.py +6 -0
  53. reconcile/openshift_base.py +47 -1
  54. reconcile/openshift_cluster_bots.py +2 -1
  55. reconcile/openshift_resources_base.py +6 -2
  56. reconcile/openshift_saas_deploy.py +2 -2
  57. reconcile/openshift_saas_deploy_trigger_cleaner.py +3 -5
  58. reconcile/openshift_upgrade_watcher.py +3 -3
  59. reconcile/queries.py +131 -0
  60. reconcile/saas_auto_promotions_manager/subscriber.py +4 -3
  61. reconcile/slack_usergroups.py +4 -3
  62. reconcile/sql_query.py +1 -0
  63. reconcile/statuspage/integrations/maintenances.py +4 -3
  64. reconcile/statuspage/status.py +5 -8
  65. reconcile/templates/rosa-classic-cluster-creation.sh.j2 +4 -0
  66. reconcile/templates/rosa-hcp-cluster-creation.sh.j2 +3 -0
  67. reconcile/templating/renderer.py +2 -1
  68. reconcile/terraform_aws_route53.py +7 -1
  69. reconcile/terraform_init/integration.py +185 -21
  70. reconcile/terraform_resources.py +11 -1
  71. reconcile/terraform_tgw_attachments.py +7 -1
  72. reconcile/terraform_users.py +7 -0
  73. reconcile/terraform_vpc_peerings.py +14 -3
  74. reconcile/terraform_vpc_resources/integration.py +7 -0
  75. reconcile/typed_queries/aws_account_tags.py +41 -0
  76. reconcile/typed_queries/saas_files.py +2 -2
  77. reconcile/utils/aggregated_list.py +4 -3
  78. reconcile/utils/aws_api.py +51 -20
  79. reconcile/utils/aws_api_typed/api.py +38 -9
  80. reconcile/utils/aws_api_typed/cloudformation.py +149 -0
  81. reconcile/utils/aws_api_typed/logs.py +73 -0
  82. reconcile/utils/datetime_util.py +67 -0
  83. reconcile/utils/differ.py +2 -3
  84. reconcile/utils/early_exit_cache.py +3 -2
  85. reconcile/utils/expiration.py +7 -3
  86. reconcile/utils/external_resource_spec.py +24 -1
  87. reconcile/utils/filtering.py +1 -1
  88. reconcile/utils/helm.py +2 -1
  89. reconcile/utils/helpers.py +1 -1
  90. reconcile/utils/jinja2/utils.py +4 -96
  91. reconcile/utils/jira_client.py +82 -63
  92. reconcile/utils/jjb_client.py +9 -12
  93. reconcile/utils/jobcontroller/controller.py +1 -1
  94. reconcile/utils/jobcontroller/models.py +17 -1
  95. reconcile/utils/json.py +32 -0
  96. reconcile/utils/merge_request_manager/merge_request_manager.py +3 -3
  97. reconcile/utils/merge_request_manager/parser.py +2 -2
  98. reconcile/utils/mr/app_interface_reporter.py +2 -2
  99. reconcile/utils/mr/base.py +2 -2
  100. reconcile/utils/mr/notificator.py +2 -2
  101. reconcile/utils/mr/update_access_report_base.py +3 -4
  102. reconcile/utils/oc.py +113 -95
  103. reconcile/utils/oc_filters.py +3 -3
  104. reconcile/utils/ocm/products.py +6 -0
  105. reconcile/utils/ocm/search_filters.py +3 -6
  106. reconcile/utils/ocm/service_log.py +3 -5
  107. reconcile/utils/openshift_resource.py +10 -5
  108. reconcile/utils/output.py +3 -2
  109. reconcile/utils/pagerduty_api.py +5 -5
  110. reconcile/utils/runtime/integration.py +1 -2
  111. reconcile/utils/runtime/runner.py +2 -2
  112. reconcile/utils/saasherder/models.py +2 -1
  113. reconcile/utils/saasherder/saasherder.py +9 -7
  114. reconcile/utils/slack_api.py +24 -2
  115. reconcile/utils/sloth.py +171 -2
  116. reconcile/utils/sqs_gateway.py +2 -1
  117. reconcile/utils/state.py +2 -1
  118. reconcile/utils/terraform_client.py +4 -3
  119. reconcile/utils/terrascript_aws_client.py +165 -111
  120. reconcile/utils/vault.py +1 -1
  121. reconcile/vault_replication.py +107 -42
  122. tools/app_interface_reporter.py +4 -4
  123. tools/cli_commands/systems_and_tools.py +5 -1
  124. tools/qontract_cli.py +25 -13
  125. {qontract_reconcile-0.10.2.dev345.dist-info → qontract_reconcile-0.10.2.dev408.dist-info}/WHEEL +0 -0
  126. {qontract_reconcile-0.10.2.dev345.dist-info → qontract_reconcile-0.10.2.dev408.dist-info}/entry_points.txt +0 -0
@@ -2,31 +2,41 @@ from __future__ import annotations
2
2
 
3
3
  import logging
4
4
  import re
5
- import typing
6
5
  from collections import defaultdict
7
6
  from datetime import UTC, datetime, timedelta
8
- from enum import Enum
9
7
  from typing import TYPE_CHECKING
10
8
 
11
- from botocore.exceptions import ClientError
12
9
  from pydantic import BaseModel
13
10
 
14
- from reconcile import queries
15
11
  from reconcile.gql_definitions.aws_cloudwatch_log_retention.aws_accounts import (
16
12
  AWSAccountCleanupOptionCloudWatchV1,
17
13
  AWSAccountV1,
18
14
  )
15
+ from reconcile.typed_queries.app_interface_vault_settings import (
16
+ get_app_interface_vault_settings,
17
+ )
18
+ from reconcile.typed_queries.aws_account_tags import get_aws_account_tags
19
19
  from reconcile.typed_queries.aws_cloudwatch_log_retention.aws_accounts import (
20
20
  get_aws_accounts,
21
21
  )
22
+ from reconcile.typed_queries.external_resources import get_settings
22
23
  from reconcile.utils import gql
23
- from reconcile.utils.aws_api import AWSApi
24
+ from reconcile.utils.aws_api_typed.api import AWSApi, AWSStaticCredentials
25
+ from reconcile.utils.datetime_util import utc_now
26
+ from reconcile.utils.differ import diff_mappings
27
+ from reconcile.utils.secret_reader import create_secret_reader
28
+ from reconcile.utils.state import init_state
29
+
30
+ TAGS_KEY = "tags.json"
24
31
 
25
32
  if TYPE_CHECKING:
26
33
  from collections.abc import Iterable
27
34
 
28
35
  from mypy_boto3_logs.type_defs import LogGroupTypeDef
29
36
 
37
+ from reconcile.utils.aws_api_typed.logs import AWSApiLogs
38
+ from reconcile.utils.gql import GqlApi
39
+
30
40
 
31
41
  QONTRACT_INTEGRATION = "aws_cloudwatch_log_retention"
32
42
  MANAGED_BY_INTEGRATION_KEY = "managed_by_integration"
@@ -35,7 +45,7 @@ DEFAULT_RETENTION_IN_DAYS = 90
35
45
 
36
46
 
37
47
  class AWSCloudwatchCleanupOption(BaseModel):
38
- regex: typing.Pattern
48
+ regex: re.Pattern
39
49
  retention_in_days: int
40
50
  delete_empty_log_group: bool
41
51
 
@@ -67,16 +77,6 @@ def get_desired_cleanup_options_by_region(
67
77
  return result
68
78
 
69
79
 
70
- def create_awsapi_client(accounts: list[AWSAccountV1], thread_pool_size: int) -> AWSApi:
71
- settings = queries.get_secret_reader_settings()
72
- return AWSApi(
73
- thread_pool_size,
74
- [account.dict(by_alias=True) for account in accounts],
75
- settings=settings,
76
- init_users=False,
77
- )
78
-
79
-
80
80
  def is_empty(log_group: LogGroupTypeDef) -> bool:
81
81
  return log_group["storedBytes"] == 0
82
82
 
@@ -85,47 +85,32 @@ def is_longer_than_retention(
85
85
  log_group: LogGroupTypeDef,
86
86
  desired_retention_days: int,
87
87
  ) -> bool:
88
- return datetime.fromtimestamp(log_group["creationTime"] / 1000, tz=UTC) + timedelta(
89
- days=desired_retention_days
90
- ) < datetime.now(tz=UTC)
91
-
92
-
93
- class TagStatus(Enum):
94
- NOT_SET = "NOT_SET"
95
- MANAGED_BY_CURRENT_INTEGRATION = "MANAGED_BY_CURRENT_INTEGRATION"
96
- MANAGED_BY_OTHER_INTEGRATION = "MANAGED_BY_OTHER_INTEGRATION"
88
+ return (
89
+ datetime.fromtimestamp(log_group["creationTime"] / 1000, tz=UTC)
90
+ + timedelta(days=desired_retention_days)
91
+ < utc_now()
92
+ )
97
93
 
98
94
 
99
- def get_tag_status(
100
- log_group: LogGroupTypeDef,
101
- account_name: str,
102
- region: str,
103
- aws_api: AWSApi,
104
- ) -> TagStatus:
105
- tags = aws_api.get_cloudwatch_log_group_tags(
106
- account_name,
107
- log_group["arn"],
108
- region,
109
- )
95
+ def _is_managed_by_other_integration(tags: dict[str, str]) -> bool:
110
96
  managed_by_integration = tags.get(MANAGED_BY_INTEGRATION_KEY)
111
- if managed_by_integration is None:
112
- return TagStatus.NOT_SET
113
- if managed_by_integration == QONTRACT_INTEGRATION:
114
- return TagStatus.MANAGED_BY_CURRENT_INTEGRATION
115
- return TagStatus.MANAGED_BY_OTHER_INTEGRATION
97
+ return (
98
+ managed_by_integration is not None
99
+ and managed_by_integration != QONTRACT_INTEGRATION
100
+ )
116
101
 
117
102
 
118
103
  def _reconcile_log_group(
119
104
  dry_run: bool,
120
- aws_log_group: LogGroupTypeDef,
105
+ log_group: LogGroupTypeDef,
121
106
  desired_cleanup_options: Iterable[AWSCloudwatchCleanupOption],
122
- account_name: str,
123
- region: str,
124
- awsapi: AWSApi,
107
+ desired_tags: dict[str, str],
108
+ last_tags: dict[str, str],
109
+ aws_api_logs: AWSApiLogs,
125
110
  ) -> None:
126
- current_retention_in_days = aws_log_group.get("retentionInDays")
127
- log_group_name = aws_log_group["logGroupName"]
128
- log_group_arn = aws_log_group["arn"]
111
+ current_retention_in_days = log_group.get("retentionInDays")
112
+ log_group_name = log_group["logGroupName"]
113
+ log_group_arn = log_group["arn"]
129
114
 
130
115
  desired_cleanup_option = _find_desired_cleanup_option(
131
116
  log_group_name, desired_cleanup_options
@@ -133,54 +118,66 @@ def _reconcile_log_group(
133
118
 
134
119
  if (
135
120
  desired_cleanup_option.delete_empty_log_group
136
- and is_empty(aws_log_group)
121
+ and is_empty(log_group)
137
122
  and is_longer_than_retention(
138
- aws_log_group, desired_cleanup_option.retention_in_days
123
+ log_group, desired_cleanup_option.retention_in_days
139
124
  )
140
125
  ):
141
- if (
142
- get_tag_status(aws_log_group, account_name, region, awsapi)
143
- != TagStatus.MANAGED_BY_OTHER_INTEGRATION
144
- ):
126
+ tags = aws_api_logs.get_tags(log_group_arn)
127
+ if not _is_managed_by_other_integration(tags):
145
128
  logging.info(
146
129
  "Deleting empty log group %s",
147
130
  log_group_arn,
148
131
  )
149
132
  if not dry_run:
150
- awsapi.delete_cloudwatch_log_group(account_name, log_group_name, region)
133
+ aws_api_logs.delete_log_group(log_group_name)
151
134
  return
152
135
 
153
- if current_retention_in_days == desired_cleanup_option.retention_in_days:
136
+ if (
137
+ current_retention_in_days == desired_cleanup_option.retention_in_days
138
+ and last_tags == desired_tags
139
+ ):
154
140
  return
155
141
 
156
- match get_tag_status(aws_log_group, account_name, region, awsapi):
157
- case TagStatus.MANAGED_BY_OTHER_INTEGRATION:
158
- return
159
- case TagStatus.MANAGED_BY_CURRENT_INTEGRATION:
160
- pass
161
- case TagStatus.NOT_SET:
162
- logging.info(
163
- "Setting tag %s for log group %s",
164
- MANAGED_TAG,
142
+ current_tags = aws_api_logs.get_tags(log_group_arn)
143
+ if _is_managed_by_other_integration(current_tags):
144
+ return
145
+
146
+ diff_result = diff_mappings(
147
+ current=current_tags,
148
+ desired=desired_tags,
149
+ )
150
+ if to_delete := diff_result.delete.keys() & last_tags.keys():
151
+ logging.info(
152
+ "Deleting tags %s for log group %s",
153
+ to_delete,
154
+ log_group_arn,
155
+ )
156
+ if not dry_run:
157
+ aws_api_logs.delete_tags(
165
158
  log_group_arn,
159
+ to_delete,
166
160
  )
167
- if not dry_run:
168
- awsapi.create_cloudwatch_tag(
169
- account_name, log_group_arn, MANAGED_TAG, region
170
- )
161
+ if diff_result.add or diff_result.change:
162
+ logging.info(
163
+ "Setting tags %s for log group %s",
164
+ desired_tags,
165
+ log_group_arn,
166
+ )
167
+ if not dry_run:
168
+ aws_api_logs.set_tags(log_group_arn, desired_tags)
171
169
 
172
- logging.info(
173
- "Setting %s retention days to %d",
174
- log_group_arn,
175
- desired_cleanup_option.retention_in_days,
176
- )
177
- if not dry_run:
178
- awsapi.set_cloudwatch_log_retention(
179
- account_name,
180
- log_group_name,
170
+ if current_retention_in_days != desired_cleanup_option.retention_in_days:
171
+ logging.info(
172
+ "Setting %s retention days to %d",
173
+ log_group_arn,
181
174
  desired_cleanup_option.retention_in_days,
182
- region,
183
175
  )
176
+ if not dry_run:
177
+ aws_api_logs.put_retention_policy(
178
+ log_group_name,
179
+ desired_cleanup_option.retention_in_days,
180
+ )
184
181
 
185
182
 
186
183
  def _find_desired_cleanup_option(
@@ -191,60 +188,63 @@ def _find_desired_cleanup_option(
191
188
  Find the first cleanup option that regex matches the log group name.
192
189
  If no match is found, return the default cleanup option.
193
190
 
194
- :param log_group_name: The name of the log group
195
- :param desired_cleanup_options: The desired cleanup options
196
- :return: The desired cleanup option
191
+ Args:
192
+ log_group_name: The name of the log group.
193
+ desired_cleanup_options: A list of desired cleanup options.
194
+ Returns:
195
+ The matching cleanup option or the default one.
197
196
  """
198
- return next(
199
- (o for o in desired_cleanup_options if o.regex.match(log_group_name)),
200
- DEFAULT_AWS_CLOUDWATCH_CLEANUP_OPTION,
201
- )
197
+ for option in desired_cleanup_options:
198
+ if option.regex.match(log_group_name):
199
+ return option
200
+ return DEFAULT_AWS_CLOUDWATCH_CLEANUP_OPTION
202
201
 
203
202
 
204
203
  def _reconcile_log_groups(
205
204
  dry_run: bool,
206
205
  aws_account: AWSAccountV1,
207
- awsapi: AWSApi,
208
- ) -> None:
209
- account_name = aws_account.name
210
- desired_cleanup_options_by_region = get_desired_cleanup_options_by_region(
211
- aws_account
206
+ last_tags: dict[str, str],
207
+ default_tags: dict[str, str],
208
+ automation_token: dict[str, str],
209
+ ) -> dict[str, str]:
210
+ desired_tags = (
211
+ default_tags | get_aws_account_tags(aws_account.organization) | MANAGED_TAG
212
212
  )
213
- try:
214
- for (
215
- region,
216
- desired_cleanup_options,
217
- ) in desired_cleanup_options_by_region.items():
218
- for aws_log_group in awsapi.get_cloudwatch_log_groups(
219
- account_name,
220
- region,
221
- ):
222
- _reconcile_log_group(
223
- dry_run=dry_run,
224
- aws_log_group=aws_log_group,
225
- desired_cleanup_options=desired_cleanup_options,
226
- account_name=account_name,
227
- region=region,
228
- awsapi=awsapi,
213
+ for (
214
+ region,
215
+ desired_cleanup_options,
216
+ ) in get_desired_cleanup_options_by_region(aws_account).items():
217
+ aws_credentials = AWSStaticCredentials(
218
+ access_key_id=automation_token["aws_access_key_id"],
219
+ secret_access_key=automation_token["aws_secret_access_key"],
220
+ region=region,
221
+ )
222
+ with AWSApi(aws_credentials) as aws_api:
223
+ aws_api_logs = aws_api.logs
224
+ try:
225
+ for log_group in aws_api_logs.get_log_groups():
226
+ _reconcile_log_group(
227
+ dry_run=dry_run,
228
+ log_group=log_group,
229
+ desired_cleanup_options=desired_cleanup_options,
230
+ desired_tags=desired_tags,
231
+ last_tags=last_tags,
232
+ aws_api_logs=aws_api_logs,
233
+ )
234
+ except aws_api_logs.client.exceptions.ClientError as e:
235
+ logging.error(
236
+ "Error reconciling log groups for %s: %s",
237
+ aws_account.name,
238
+ e,
229
239
  )
230
- except ClientError as e:
231
- if e.response["Error"]["Code"] == "AccessDeniedException":
232
- logging.info(
233
- "Access denied for aws account %s. Skipping...",
234
- account_name,
235
- )
236
- else:
237
- logging.error(
238
- "Error reconciling log groups for %s: %s",
239
- account_name,
240
- e,
241
- )
240
+ return last_tags
241
+ return desired_tags
242
242
 
243
243
 
244
- def get_active_aws_accounts() -> list[AWSAccountV1]:
244
+ def get_active_aws_accounts(gql_api: GqlApi) -> list[AWSAccountV1]:
245
245
  return [
246
246
  account
247
- for account in get_aws_accounts(gql.get_api())
247
+ for account in get_aws_accounts(gql_api)
248
248
  if not (
249
249
  account.disable
250
250
  and account.disable.integrations
@@ -253,8 +253,37 @@ def get_active_aws_accounts() -> list[AWSAccountV1]:
253
253
  ]
254
254
 
255
255
 
256
- def run(dry_run: bool, thread_pool_size: int) -> None:
257
- aws_accounts = get_active_aws_accounts()
258
- with create_awsapi_client(aws_accounts, thread_pool_size) as awsapi:
259
- for aws_account in aws_accounts:
260
- _reconcile_log_groups(dry_run, aws_account, awsapi)
256
+ def get_default_tags(gql_api: GqlApi) -> dict[str, str]:
257
+ try:
258
+ return get_settings(gql_api.query).default_tags
259
+ except ValueError:
260
+ # no settings found
261
+ return {}
262
+
263
+
264
+ def run(dry_run: bool) -> None:
265
+ gql_api = gql.get_api()
266
+ aws_accounts = get_active_aws_accounts(gql_api)
267
+ vault_settings = get_app_interface_vault_settings(query_func=gql_api.query)
268
+ secret_reader = create_secret_reader(use_vault=vault_settings.vault)
269
+ default_tags = get_default_tags(gql_api)
270
+
271
+ with init_state(
272
+ integration=QONTRACT_INTEGRATION,
273
+ secret_reader=secret_reader,
274
+ ) as state:
275
+ last_tags = state.get(TAGS_KEY, {})
276
+ desired_tags = {
277
+ aws_account.name: _reconcile_log_groups(
278
+ dry_run=dry_run,
279
+ aws_account=aws_account,
280
+ last_tags=last_tags.get(aws_account.name, {}),
281
+ default_tags=default_tags,
282
+ automation_token=secret_reader.read_all_secret(
283
+ aws_account.automation_token
284
+ ),
285
+ )
286
+ for aws_account in aws_accounts
287
+ }
288
+ if not dry_run and desired_tags != last_tags:
289
+ state.add(TAGS_KEY, desired_tags, force=True)
@@ -1,11 +1,11 @@
1
1
  import base64
2
- import json
3
2
  import logging
4
3
  from collections.abc import Mapping
5
4
  from typing import Any
6
5
 
7
6
  from reconcile import queries
8
7
  from reconcile.utils.aws_api import AWSApi
8
+ from reconcile.utils.json import json_dumps
9
9
  from reconcile.utils.vault import VaultClient
10
10
 
11
11
  QONTRACT_INTEGRATION = "aws-ecr-image-pull-secrets"
@@ -35,7 +35,7 @@ def construct_dockercfg_secret_data(data: Mapping[str, Any]) -> dict[str, str]:
35
35
  }
36
36
  }
37
37
 
38
- return {".dockerconfigjson": enc_dec(json.dumps(data))}
38
+ return {".dockerconfigjson": enc_dec(json_dumps(data))}
39
39
 
40
40
 
41
41
  def construct_basic_auth_secret_data(data: Mapping[str, Any]) -> dict[str, str]:
reconcile/aws_iam_keys.py CHANGED
@@ -65,6 +65,7 @@ def init_tf_working_dirs(
65
65
  thread_pool_size,
66
66
  accounts,
67
67
  settings=settings,
68
+ default_tags=None,
68
69
  )
69
70
  return ts.dump()
70
71
 
@@ -19,6 +19,7 @@ from reconcile.gql_definitions.aws_saml_idp.aws_accounts import (
19
19
  query as aws_accounts_query,
20
20
  )
21
21
  from reconcile.status import ExitCodes
22
+ from reconcile.typed_queries.external_resources import get_settings
22
23
  from reconcile.utils import gql
23
24
  from reconcile.utils.aws_api import AWSApi
24
25
  from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
@@ -125,13 +126,18 @@ class AwsSamlIdpIntegration(QontractReconcileIntegration[AwsSamlIdpIntegrationPa
125
126
  gql_api.query, account_name=self.params.account_name
126
127
  )
127
128
  aws_accounts_dict = [account.dict(by_alias=True) for account in aws_accounts]
128
-
129
+ try:
130
+ default_tags = get_settings().default_tags
131
+ except ValueError:
132
+ # no external resources settings found
133
+ default_tags = None
129
134
  ts = TerrascriptClient(
130
135
  self.name.replace("-", "_"),
131
136
  "",
132
137
  self.params.thread_pool_size,
133
138
  aws_accounts_dict,
134
139
  secret_reader=self.secret_reader,
140
+ default_tags=default_tags,
135
141
  )
136
142
 
137
143
  for saml_idp_config in self.build_saml_idp_config(
@@ -1,4 +1,3 @@
1
- import json
2
1
  import logging
3
2
  import sys
4
3
  from collections.abc import (
@@ -22,6 +21,7 @@ from reconcile.gql_definitions.aws_saml_roles.roles import (
22
21
  query as roles_query,
23
22
  )
24
23
  from reconcile.status import ExitCodes
24
+ from reconcile.typed_queries.external_resources import get_settings
25
25
  from reconcile.utils import gql
26
26
  from reconcile.utils.aws_api import AWSApi
27
27
  from reconcile.utils.aws_helper import unique_sso_aws_accounts
@@ -32,6 +32,7 @@ from reconcile.utils.extended_early_exit import (
32
32
  ExtendedEarlyExitRunnerResult,
33
33
  extended_early_exit_run,
34
34
  )
35
+ from reconcile.utils.json import json_dumps
35
36
  from reconcile.utils.runtime.integration import (
36
37
  PydanticRunParams,
37
38
  QontractReconcileIntegration,
@@ -87,7 +88,7 @@ class CustomPolicy(BaseModel):
87
88
 
88
89
  See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html
89
90
  """
90
- if len(json.dumps(v, separators=(",", ":"))) > 6144:
91
+ if len(json_dumps(v, compact=True)) > 6144:
91
92
  raise ValueError(
92
93
  f"The policy document '{v}' is too large. AWS policy documents must be 6144 characters or less (w/o white spaces)."
93
94
  )
@@ -253,13 +254,18 @@ class AwsSamlRolesIntegration(
253
254
  )
254
255
  aws_accounts_dict = [account.dict(by_alias=True) for account in aws_accounts]
255
256
  aws_roles = self.get_roles(gql_api.query, account_name=self.params.account_name)
256
-
257
+ try:
258
+ default_tags = get_settings().default_tags
259
+ except ValueError:
260
+ # no external resources settings found
261
+ default_tags = None
257
262
  ts = TerrascriptClient(
258
263
  self.name.replace("-", "_"),
259
264
  "",
260
265
  self.params.thread_pool_size,
261
266
  aws_accounts_dict,
262
267
  secret_reader=self.secret_reader,
268
+ default_tags=default_tags,
263
269
  )
264
270
  self.populate_saml_iam_roles(ts, aws_roles)
265
271
  working_dirs = ts.dump(print_to_file=self.params.print_to_file)
@@ -140,7 +140,7 @@ def write_coverage_report_to_mr(
140
140
  approver_reachability = set()
141
141
  for d in change_decisions:
142
142
  approvers = [
143
- f"{cr.context} - {' '.join([f'@{a.org_username}' if a.tag_on_merge_requests else a.org_username for a in cr.approvers])}"
143
+ f"{cr.context} - {' '.join([f'@{a.org_username}' if (a.tag_on_merge_requests or len(cr.approvers) == 1) else a.org_username for a in cr.approvers])}"
144
144
  for cr in d.change_responsibles
145
145
  ]
146
146
  if d.coverable_by_fragment_decisions:
@@ -1,5 +1,4 @@
1
1
  import copy
2
- import json
3
2
  from dataclasses import dataclass
4
3
  from enum import Enum
5
4
  from functools import reduce
@@ -11,6 +10,7 @@ from deepdiff.helper import CannotCompare
11
10
  from deepdiff.model import DiffLevel
12
11
  from deepdiff.path import parse_path
13
12
 
13
+ from reconcile.utils.json import json_dumps
14
14
  from reconcile.utils.jsonpath import parse_jsonpath
15
15
 
16
16
 
@@ -75,7 +75,7 @@ class Diff:
75
75
  def _value_repr(self, value: Any | None) -> str | None:
76
76
  if value:
77
77
  if isinstance(value, dict | list):
78
- return json.dumps(value, indent=2)
78
+ return json_dumps(value, indent=2)
79
79
  return str(value)
80
80
  return value
81
81
 
@@ -251,8 +251,6 @@ def deepdiff_path_to_jsonpath(deep_diff_path: str) -> jsonpath_ng.JSONPath:
251
251
  case int():
252
252
  return jsonpath_ng.Index(element)
253
253
  case str():
254
- if "." in element:
255
- return jsonpath_ng.Fields(f"'{element}'")
256
254
  return jsonpath_ng.Fields(element)
257
255
 
258
256
  path_parts = [build_jsonpath_part(p) for p in parse_path(deep_diff_path)]
reconcile/checkpoint.py CHANGED
@@ -26,6 +26,7 @@ from jira import Issue
26
26
 
27
27
  from reconcile.utils.constants import PROJ_ROOT
28
28
  from reconcile.utils.jira_client import JiraClient
29
+ from reconcile.utils.secret_reader import SecretReaderBase
29
30
 
30
31
  DEFAULT_CHECKPOINT_LABELS = ("sre-checkpoint",)
31
32
 
@@ -118,8 +119,8 @@ def file_ticket(
118
119
  def report_invalid_metadata(
119
120
  app: Mapping[str, Any],
120
121
  path: str,
121
- board: Mapping[str, str | Mapping],
122
- settings: Mapping[str, Any],
122
+ board: Mapping[str, Any],
123
+ secret_reader: SecretReaderBase,
123
124
  parent: str,
124
125
  dry_run: bool = False,
125
126
  ) -> None:
@@ -150,7 +151,14 @@ def report_invalid_metadata(
150
151
  path=path,
151
152
  )
152
153
  else:
153
- jira = JiraClient(board, settings)
154
+ jira = JiraClient.create(
155
+ project_name=board["name"],
156
+ token=secret_reader.read_secret(board["server"]["token"]),
157
+ email=secret_reader.read_secret(board["server"]["email"])
158
+ if board["server"]["email"]
159
+ else None,
160
+ server_url=board["server"]["server_url"],
161
+ )
154
162
  do_cut = partial(
155
163
  file_ticket, # type: ignore
156
164
  jira=jira,
reconcile/cli.py CHANGED
@@ -1,6 +1,5 @@
1
1
  # ruff: noqa: PLC0415 - `import` should be at the top-level of a file
2
2
  import faulthandler
3
- import json
4
3
  import logging
5
4
  import os
6
5
  import re
@@ -31,6 +30,7 @@ from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
31
30
  from reconcile.utils.exceptions import PrintToFileInGitRepositoryError
32
31
  from reconcile.utils.git import is_file_in_git_repo
33
32
  from reconcile.utils.gql import GqlApiSingleton
33
+ from reconcile.utils.json import json_dumps
34
34
  from reconcile.utils.promtool import PROMTOOL_VERSION, PROMTOOL_VERSION_REGEX
35
35
  from reconcile.utils.runtime.environment import init_env
36
36
  from reconcile.utils.runtime.integration import (
@@ -608,7 +608,7 @@ def run_class_integration(
608
608
  if dump_schemas_file:
609
609
  gqlapi = gql.get_api()
610
610
  with open(dump_schemas_file, "w", encoding="locale") as f:
611
- f.write(json.dumps(gqlapi.get_queried_schemas()))
611
+ f.write(json_dumps(gqlapi.get_queried_schemas()))
612
612
 
613
613
 
614
614
  @click.group()
@@ -1028,7 +1028,7 @@ def aws_account_manager(
1028
1028
  "--state-tmpl-resource",
1029
1029
  help="Resource name of the state template-collection template in the app-interface.",
1030
1030
  required=True,
1031
- default="/terraform-init/terraform-state.yml",
1031
+ default="/terraform-init/terraform-state.yml.j2",
1032
1032
  )
1033
1033
  @click.option(
1034
1034
  "--template-collection-root-path",
@@ -1036,12 +1036,26 @@ def aws_account_manager(
1036
1036
  required=True,
1037
1037
  default="data/templating/collections/terraform-init",
1038
1038
  )
1039
+ @click.option(
1040
+ "--cloudformation-template-resource",
1041
+ help="Resource name of the CloudFormation template to create the S3 bucket",
1042
+ required=True,
1043
+ default="/terraform-init/terraform-state-s3-bucket.yaml",
1044
+ )
1045
+ @click.option(
1046
+ "--cloudformation-import-template-resource",
1047
+ help="Resource name of the CloudFormation template to import existing S3 bucket",
1048
+ required=True,
1049
+ default="/terraform-init/terraform-state-s3-bucket-import.yaml",
1050
+ )
1039
1051
  @click.pass_context
1040
1052
  def terraform_init(
1041
1053
  ctx: click.Context,
1042
1054
  account_name: str | None,
1043
1055
  state_tmpl_resource: str,
1044
1056
  template_collection_root_path: str,
1057
+ cloudformation_template_resource: str,
1058
+ cloudformation_import_template_resource: str,
1045
1059
  ) -> None:
1046
1060
  from reconcile.terraform_init.integration import (
1047
1061
  TerraformInitIntegration,
@@ -1054,6 +1068,8 @@ def terraform_init(
1054
1068
  account_name=account_name,
1055
1069
  state_tmpl_resource=state_tmpl_resource,
1056
1070
  template_collection_root_path=template_collection_root_path,
1071
+ cloudformation_template_resource=cloudformation_template_resource,
1072
+ cloudformation_import_template_resource=cloudformation_import_template_resource,
1057
1073
  )
1058
1074
  ),
1059
1075
  ctx=ctx,
@@ -1135,9 +1151,17 @@ def jenkins_webhooks_cleaner(ctx: click.Context) -> None:
1135
1151
  "--jira-board-name", help="The Jira board to act on.", default=None, multiple=True
1136
1152
  )
1137
1153
  @click.option("--board-check-interval", help="Check interval in minutes", default=120)
1154
+ @click.option(
1155
+ "--use-cache/--no-use-cache",
1156
+ default=True,
1157
+ help="Use cached results for validation.",
1158
+ )
1138
1159
  @click.pass_context
1139
1160
  def jira_permissions_validator(
1140
- ctx: click.Context, jira_board_name: Iterable[str] | None, board_check_interval: int
1161
+ ctx: click.Context,
1162
+ jira_board_name: Iterable[str] | None,
1163
+ board_check_interval: int,
1164
+ use_cache: bool,
1141
1165
  ) -> None:
1142
1166
  import reconcile.jira_permissions_validator
1143
1167
 
@@ -1146,6 +1170,7 @@ def jira_permissions_validator(
1146
1170
  ctx,
1147
1171
  jira_board_name=jira_board_name,
1148
1172
  board_check_interval_sec=board_check_interval * 60,
1173
+ use_cache=use_cache,
1149
1174
  )
1150
1175
 
1151
1176
 
@@ -1270,14 +1295,14 @@ def aws_ami_cleanup(ctx: click.Context, thread_pool_size: int) -> None:
1270
1295
  run_integration(reconcile.aws_ami_cleanup.integration, ctx, thread_pool_size)
1271
1296
 
1272
1297
 
1273
- @integration.command(short_help="Set up retention period for Cloudwatch logs.")
1274
- @threaded()
1298
+ @integration.command(short_help="Set up retention period and tags for Cloudwatch logs.")
1275
1299
  @click.pass_context
1276
- def aws_cloudwatch_log_retention(ctx: click.Context, thread_pool_size: int) -> None:
1300
+ def aws_cloudwatch_log_retention(ctx: click.Context) -> None:
1277
1301
  import reconcile.aws_cloudwatch_log_retention.integration
1278
1302
 
1279
1303
  run_integration(
1280
- reconcile.aws_cloudwatch_log_retention.integration, ctx, thread_pool_size
1304
+ reconcile.aws_cloudwatch_log_retention.integration,
1305
+ ctx,
1281
1306
  )
1282
1307
 
1283
1308