qontract-reconcile 0.10.2.dev159__py3-none-any.whl → 0.10.2.dev161__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {qontract_reconcile-0.10.2.dev159.dist-info → qontract_reconcile-0.10.2.dev161.dist-info}/METADATA +1 -1
- {qontract_reconcile-0.10.2.dev159.dist-info → qontract_reconcile-0.10.2.dev161.dist-info}/RECORD +15 -8
- reconcile/aws_cloudwatch_log_retention/integration.py +39 -25
- reconcile/cli.py +4 -6
- reconcile/gcp_image_mirror.py +276 -0
- reconcile/gql_definitions/aws_cloudwatch_log_retention/__init__.py +0 -0
- reconcile/gql_definitions/aws_cloudwatch_log_retention/aws_accounts.py +158 -0
- reconcile/gql_definitions/fragments/container_image_mirror.py +33 -0
- reconcile/gql_definitions/gcp/__init__.py +0 -0
- reconcile/gql_definitions/gcp/gcp_docker_repos.py +128 -0
- reconcile/gql_definitions/gcp/gcp_projects.py +77 -0
- reconcile/gql_definitions/introspection.json +131 -1
- reconcile/typed_queries/aws_cloudwatch_log_retention/aws_accounts.py +12 -0
- reconcile/gcr_mirror.py +0 -278
- {qontract_reconcile-0.10.2.dev159.dist-info → qontract_reconcile-0.10.2.dev161.dist-info}/WHEEL +0 -0
- {qontract_reconcile-0.10.2.dev159.dist-info → qontract_reconcile-0.10.2.dev161.dist-info}/entry_points.txt +0 -0
{qontract_reconcile-0.10.2.dev159.dist-info → qontract_reconcile-0.10.2.dev161.dist-info}/METADATA
RENAMED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: qontract-reconcile
|
3
|
-
Version: 0.10.2.
|
3
|
+
Version: 0.10.2.dev161
|
4
4
|
Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
|
5
5
|
Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
|
6
6
|
Project-URL: repository, https://github.com/app-sre/qontract-reconcile
|
{qontract_reconcile-0.10.2.dev159.dist-info → qontract_reconcile-0.10.2.dev161.dist-info}/RECORD
RENAMED
@@ -10,7 +10,7 @@ reconcile/aws_iam_password_reset.py,sha256=O0JX2N5kNRKs3u2xzu4NNrI6p0ag5JWy3MTsv
|
|
10
10
|
reconcile/aws_support_cases_sos.py,sha256=PDhilxQ4TBxVnxUPIUdTbKEaNUI0wzPiEsB91oHT2fY,3384
|
11
11
|
reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=O1wFp52EyF538c6txaWBs8eMtUIy19gyHZ6VzJ6QXS8,3512
|
12
12
|
reconcile/checkpoint.py,sha256=_JhMxrye5BgkRMxWYuf7Upli6XayPINKSsuo3ynHTRc,5010
|
13
|
-
reconcile/cli.py,sha256=
|
13
|
+
reconcile/cli.py,sha256=xyVnxNyq3IPISWwFlB9j4HAFjowXYv3EdsEGIMFhTy0,108438
|
14
14
|
reconcile/closedbox_endpoint_monitoring_base.py,sha256=al7m8EgnnYx90rY1REryW3byN_ItfJfAzEeLtjbCfi0,4921
|
15
15
|
reconcile/cluster_deployment_mapper.py,sha256=5gumAaRCcFXsabUJ1dnuUy9WrP_FEEM5JnOnE8ch9sE,2326
|
16
16
|
reconcile/dashdotdb_base.py,sha256=83ZWIf5JJk3P_D69y2TmXRcQr6ELJGlv10OM0h7fJVs,4767
|
@@ -22,7 +22,7 @@ reconcile/database_access_manager.py,sha256=Z3aAmw2LsmMIIor-bOGzziVZdVNC82Gmw8oH
|
|
22
22
|
reconcile/deadmanssnitch.py,sha256=n-5W-djUgwzpmdDM4eQIZpkkDmHY0vndt-42LJXI4Y8,7491
|
23
23
|
reconcile/email_sender.py,sha256=38Wvl6WHqCwlqLx4oxVJOIeDmoJsyitD3g1F4jTkAj8,4246
|
24
24
|
reconcile/gabi_authorized_users.py,sha256=Jwvo97nzUX3NIl2VHKuZlT0-I40qk2VnACbafe91T2o,4854
|
25
|
-
reconcile/
|
25
|
+
reconcile/gcp_image_mirror.py,sha256=M5pimd0j13BBWCa8vX0fftWO0pHBfQCATIIpOGggDSA,10332
|
26
26
|
reconcile/github_org.py,sha256=Wc5cZamatuWsW2ZJT2ib5ps8l3iY3RXHwNUxVJerqz0,14173
|
27
27
|
reconcile/github_owners.py,sha256=viE1KJ-zaTxuZ5yItg2C263J0brn-Q-3hR_DkYDMbhY,3122
|
28
28
|
reconcile/github_repo_invites.py,sha256=U9UCzNVwrZ7MqODtFah8ogH0NNY-XjBin7G9gqHtCUY,2690
|
@@ -152,7 +152,7 @@ reconcile/aws_account_manager/utils.py,sha256=iYPPOtbZ7FiKkz9v5f1YXRIHw5YFOtSavU
|
|
152
152
|
reconcile/aws_ami_cleanup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
153
153
|
reconcile/aws_ami_cleanup/integration.py,sha256=KG7g9NpbKmoaveDD3oi9SinqUE29NaM-4lGo-6YuHlM,9302
|
154
154
|
reconcile/aws_cloudwatch_log_retention/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
155
|
-
reconcile/aws_cloudwatch_log_retention/integration.py,sha256=
|
155
|
+
reconcile/aws_cloudwatch_log_retention/integration.py,sha256=QY5EtCpcMN0TgOQInIDW65wT1YksMBFkK8aNK5cg-XA,8107
|
156
156
|
reconcile/aws_saml_idp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
157
157
|
reconcile/aws_saml_idp/integration.py,sha256=Z2JtUx2YIbkn0KVrVa2CoAErPB8vTykOOkWD_ZPoB94,6511
|
158
158
|
reconcile/aws_saml_roles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -227,7 +227,7 @@ reconcile/glitchtip_project_alerts/integration.py,sha256=BgMx-NyV9mTuv7Sotb2OioC
|
|
227
227
|
reconcile/glitchtip_project_dsn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
228
228
|
reconcile/glitchtip_project_dsn/integration.py,sha256=2iugub-kHYkHNK33n0v9_TeWonuxCPah_VkoTPvaajE,8077
|
229
229
|
reconcile/gql_definitions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
230
|
-
reconcile/gql_definitions/introspection.json,sha256=
|
230
|
+
reconcile/gql_definitions/introspection.json,sha256=wNGZv8V6ivviCcwPLw34nzTR5fgwKRyE68Z93HbmBBs,2296576
|
231
231
|
reconcile/gql_definitions/acs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
232
232
|
reconcile/gql_definitions/acs/acs_instances.py,sha256=L91WW9LbhJbBSrECqShQpFtjoBOsmNIYLRpMbx1io5o,2181
|
233
233
|
reconcile/gql_definitions/acs/acs_policies.py,sha256=bN5i4mks10Z23KJSj7jqp966Osq2dps4d-sPH9gjxEA,7008
|
@@ -246,6 +246,8 @@ reconcile/gql_definitions/aws_account_manager/__init__.py,sha256=47DEQpj8HBSa-_T
|
|
246
246
|
reconcile/gql_definitions/aws_account_manager/aws_accounts.py,sha256=vF51KrY2gwX0J9vESiaRMPQqdAMEtz9f_tBq52bInp0,5148
|
247
247
|
reconcile/gql_definitions/aws_ami_cleanup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
248
248
|
reconcile/gql_definitions/aws_ami_cleanup/aws_accounts.py,sha256=jIgOa888MYLLvVsn1ir3nbkhWLG5T6dBg7oDnp1q8BI,4108
|
249
|
+
reconcile/gql_definitions/aws_cloudwatch_log_retention/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
250
|
+
reconcile/gql_definitions/aws_cloudwatch_log_retention/aws_accounts.py,sha256=Am6y5LZIBncH9u7vwU48WRksnwGljpoS-xbP7MJA8-4,4976
|
249
251
|
reconcile/gql_definitions/aws_saml_idp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
250
252
|
reconcile/gql_definitions/aws_saml_idp/aws_accounts.py,sha256=pR9Qm6P9Roe4OJaDXvfm8AcfkSSAtriQdlwLwW7UdUU,2666
|
251
253
|
reconcile/gql_definitions/aws_saml_roles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -333,6 +335,7 @@ reconcile/gql_definitions/fragments/aws_infra_management_account.py,sha256=uAmAL
|
|
333
335
|
reconcile/gql_definitions/fragments/aws_vpc.py,sha256=T2egTwi2Rb0IRBBmsyag8xKpu_m6GbIAy80fhZNZwk8,1434
|
334
336
|
reconcile/gql_definitions/fragments/aws_vpc_request.py,sha256=o0qUsPrFXs8GAbtgMXQmIJxc1mw5skSIzCcidE857g8,2460
|
335
337
|
reconcile/gql_definitions/fragments/aws_vpc_request_subnet.py,sha256=qaTFT8cGzEslw51nUeb45Nfnv6kFxUm4CWrRR3xfBvA,760
|
338
|
+
reconcile/gql_definitions/fragments/container_image_mirror.py,sha256=qyfQlnKUCzFEPgUJ9VGmDYFmiGHR7VZ_YJNd4KeoolM,968
|
336
339
|
reconcile/gql_definitions/fragments/deplopy_resources.py,sha256=0u3xYqL5NpMf149BJLfPhHqAOWu06aLULdNk_2Mulxg,1089
|
337
340
|
reconcile/gql_definitions/fragments/disable.py,sha256=Ojw98OSxcovrtmw_aAyhaVHhIa1MSUbBfKX4i2IpI74,715
|
338
341
|
reconcile/gql_definitions/fragments/email_service.py,sha256=0wKpICsg4pcMfr2lszvnqbuPX7wVYoJ5cYFU2uQkHbY,803
|
@@ -353,6 +356,9 @@ reconcile/gql_definitions/fragments/terraform_state.py,sha256=S5QuTR9YlvUObiU7he
|
|
353
356
|
reconcile/gql_definitions/fragments/upgrade_policy.py,sha256=cVza8zfra1E3yBsHiS-hKbys17fvv572GFnKshJjluE,1246
|
354
357
|
reconcile/gql_definitions/fragments/user.py,sha256=TZyFEs1fBg5PkvWdyCxFDZ_3aRhcQzusfhObXFiOU_0,1025
|
355
358
|
reconcile/gql_definitions/fragments/vault_secret.py,sha256=8xoQJNx1jKw_1yradq1iLEYWzuOHra1bEHHU7WHKxqo,833
|
359
|
+
reconcile/gql_definitions/gcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
360
|
+
reconcile/gql_definitions/gcp/gcp_docker_repos.py,sha256=HvNaJxQYNPBTDmk26cOUY5_C5oBfau4bdfuI-L1Vcps,3338
|
361
|
+
reconcile/gql_definitions/gcp/gcp_projects.py,sha256=7LslYFSN2r8vYhYdi4s-zOkIrqpqyt4ayxbPNMie9M4,2108
|
356
362
|
reconcile/gql_definitions/gitlab_members/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
357
363
|
reconcile/gql_definitions/gitlab_members/gitlab_instances.py,sha256=oYPvfiOsPTGHXQeSfxXvBuvJFrwp0VtE2F0lVKFQMoU,2206
|
358
364
|
reconcile/gql_definitions/gitlab_members/permissions.py,sha256=Qzj3Fpv7xj8v9eygeP312nHRNg8er8XMRBveynPIyQM,3302
|
@@ -575,6 +581,7 @@ reconcile/typed_queries/vault.py,sha256=lkRsmobykorof3fcrIPLz-NgvAiSOWSOZc_jXBln
|
|
575
581
|
reconcile/typed_queries/app_interface_metrics_exporter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
576
582
|
reconcile/typed_queries/app_interface_metrics_exporter/onboarding_status.py,sha256=X-N1WJGOL6OR9940P0_K4-YJzkL5Vg4favhYrBxXD9A,327
|
577
583
|
reconcile/typed_queries/app_interface_metrics_exporter/terraform_repo.py,sha256=r-nJ5CucAOE_cwxnbVp5lmAAfHBG8t1h2tVmhviVYls,290
|
584
|
+
reconcile/typed_queries/aws_cloudwatch_log_retention/aws_accounts.py,sha256=WiQ84vEZp-oYvD4CVZaFDYcMBn-pkO3slwsHIQxvHks,296
|
578
585
|
reconcile/typed_queries/cost_report/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
579
586
|
reconcile/typed_queries/cost_report/app_names.py,sha256=HMEMIqAbMyVQfoQ5YXTXE4xDt7FaXBRz0QIHnsIZC1c,478
|
580
587
|
reconcile/typed_queries/cost_report/cost_namespaces.py,sha256=1GUjWXQj7U2djVHBPYcd8Cy2-enKXf0-GaplLi8JZw4,1178
|
@@ -797,7 +804,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
|
|
797
804
|
tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
|
798
805
|
tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
|
799
806
|
tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
|
800
|
-
qontract_reconcile-0.10.2.
|
801
|
-
qontract_reconcile-0.10.2.
|
802
|
-
qontract_reconcile-0.10.2.
|
803
|
-
qontract_reconcile-0.10.2.
|
807
|
+
qontract_reconcile-0.10.2.dev161.dist-info/METADATA,sha256=8F0IJKYcSCyZq47ZOd2Bn6zFBeo2I83momD5TwvCTSg,24627
|
808
|
+
qontract_reconcile-0.10.2.dev161.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
809
|
+
qontract_reconcile-0.10.2.dev161.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
|
810
|
+
qontract_reconcile-0.10.2.dev161.dist-info/RECORD,,
|
@@ -5,13 +5,22 @@ from collections import defaultdict
|
|
5
5
|
from collections.abc import Iterable
|
6
6
|
from datetime import UTC, datetime, timedelta
|
7
7
|
from enum import Enum
|
8
|
-
from typing import
|
8
|
+
from typing import (
|
9
|
+
TYPE_CHECKING,
|
10
|
+
)
|
9
11
|
|
10
12
|
from botocore.exceptions import ClientError
|
11
13
|
from pydantic import BaseModel
|
12
14
|
|
13
15
|
from reconcile import queries
|
14
|
-
from reconcile.
|
16
|
+
from reconcile.gql_definitions.aws_cloudwatch_log_retention.aws_accounts import (
|
17
|
+
AWSAccountCleanupOptionCloudWatchV1,
|
18
|
+
AWSAccountV1,
|
19
|
+
)
|
20
|
+
from reconcile.typed_queries.aws_cloudwatch_log_retention.aws_accounts import (
|
21
|
+
get_aws_accounts,
|
22
|
+
)
|
23
|
+
from reconcile.utils import gql
|
15
24
|
from reconcile.utils.aws_api import AWSApi
|
16
25
|
|
17
26
|
if TYPE_CHECKING:
|
@@ -39,31 +48,33 @@ DEFAULT_AWS_CLOUDWATCH_CLEANUP_OPTION = AWSCloudwatchCleanupOption(
|
|
39
48
|
|
40
49
|
|
41
50
|
def get_desired_cleanup_options_by_region(
|
42
|
-
account:
|
51
|
+
account: AWSAccountV1,
|
43
52
|
) -> dict[str, list[AWSCloudwatchCleanupOption]]:
|
44
|
-
default_region = account
|
53
|
+
default_region = account.resources_default_region
|
45
54
|
result = defaultdict(list)
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
delete_empty_log_group=bool(
|
55
|
-
cleanup_option["delete_empty_log_group"]
|
56
|
-
),
|
57
|
-
)
|
55
|
+
for cleanup_option in account.cleanup or []:
|
56
|
+
if isinstance(cleanup_option, AWSAccountCleanupOptionCloudWatchV1):
|
57
|
+
region = cleanup_option.region or default_region
|
58
|
+
result[region].append(
|
59
|
+
AWSCloudwatchCleanupOption(
|
60
|
+
regex=re.compile(cleanup_option.regex),
|
61
|
+
retention_in_days=cleanup_option.retention_in_days,
|
62
|
+
delete_empty_log_group=bool(cleanup_option.delete_empty_log_group),
|
58
63
|
)
|
64
|
+
)
|
59
65
|
if not result:
|
60
66
|
result[default_region].append(DEFAULT_AWS_CLOUDWATCH_CLEANUP_OPTION)
|
61
67
|
return result
|
62
68
|
|
63
69
|
|
64
|
-
def create_awsapi_client(accounts: list, thread_pool_size: int) -> AWSApi:
|
70
|
+
def create_awsapi_client(accounts: list[AWSAccountV1], thread_pool_size: int) -> AWSApi:
|
65
71
|
settings = queries.get_secret_reader_settings()
|
66
|
-
return AWSApi(
|
72
|
+
return AWSApi(
|
73
|
+
thread_pool_size,
|
74
|
+
[account.dict(by_alias=True) for account in accounts],
|
75
|
+
settings=settings,
|
76
|
+
init_users=False,
|
77
|
+
)
|
67
78
|
|
68
79
|
|
69
80
|
def is_empty(log_group: LogGroupTypeDef) -> bool:
|
@@ -192,10 +203,10 @@ def _find_desired_cleanup_option(
|
|
192
203
|
|
193
204
|
def _reconcile_log_groups(
|
194
205
|
dry_run: bool,
|
195
|
-
aws_account:
|
206
|
+
aws_account: AWSAccountV1,
|
196
207
|
awsapi: AWSApi,
|
197
208
|
) -> None:
|
198
|
-
account_name = aws_account
|
209
|
+
account_name = aws_account.name
|
199
210
|
desired_cleanup_options_by_region = get_desired_cleanup_options_by_region(
|
200
211
|
aws_account
|
201
212
|
)
|
@@ -230,12 +241,15 @@ def _reconcile_log_groups(
|
|
230
241
|
)
|
231
242
|
|
232
243
|
|
233
|
-
def get_active_aws_accounts() -> list[
|
244
|
+
def get_active_aws_accounts() -> list[AWSAccountV1]:
|
234
245
|
return [
|
235
|
-
|
236
|
-
for
|
237
|
-
if
|
238
|
-
|
246
|
+
account
|
247
|
+
for account in get_aws_accounts(gql.get_api())
|
248
|
+
if not (
|
249
|
+
account.disable
|
250
|
+
and account.disable.integrations
|
251
|
+
and "aws-cloudwatch-log-retention" in account.disable.integrations
|
252
|
+
)
|
239
253
|
]
|
240
254
|
|
241
255
|
|
reconcile/cli.py
CHANGED
@@ -1849,15 +1849,13 @@ def quay_membership(ctx):
|
|
1849
1849
|
run_integration(reconcile.quay_membership, ctx.obj)
|
1850
1850
|
|
1851
1851
|
|
1852
|
-
@integration.command(
|
1853
|
-
short_help="Mirrors external images into Google Container Registry."
|
1854
|
-
)
|
1852
|
+
@integration.command(short_help="Mirrors external images into GCP Artifact Registry.")
|
1855
1853
|
@click.pass_context
|
1856
1854
|
@binary(["skopeo"])
|
1857
|
-
def
|
1858
|
-
import reconcile.
|
1855
|
+
def gcp_image_mirror(ctx):
|
1856
|
+
import reconcile.gcp_image_mirror
|
1859
1857
|
|
1860
|
-
run_integration(reconcile.
|
1858
|
+
run_integration(reconcile.gcp_image_mirror, ctx.obj)
|
1861
1859
|
|
1862
1860
|
|
1863
1861
|
@integration.command(short_help="Mirrors external images into Quay.")
|
@@ -0,0 +1,276 @@
|
|
1
|
+
import base64
|
2
|
+
import logging
|
3
|
+
import os
|
4
|
+
import re
|
5
|
+
import tempfile
|
6
|
+
import time
|
7
|
+
from typing import Any, Self
|
8
|
+
|
9
|
+
import requests
|
10
|
+
from pydantic import BaseModel
|
11
|
+
from sretoolbox.container import (
|
12
|
+
Image,
|
13
|
+
Skopeo,
|
14
|
+
)
|
15
|
+
from sretoolbox.container.image import ImageComparisonError
|
16
|
+
from sretoolbox.container.skopeo import SkopeoCmdError
|
17
|
+
|
18
|
+
import reconcile.gql_definitions.gcp.gcp_docker_repos as gql_gcp_repos
|
19
|
+
import reconcile.gql_definitions.gcp.gcp_projects as gql_gcp_projects
|
20
|
+
from reconcile import queries
|
21
|
+
from reconcile.gql_definitions.fragments.container_image_mirror import (
|
22
|
+
ContainerImageMirror,
|
23
|
+
)
|
24
|
+
from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
|
25
|
+
from reconcile.utils import gql
|
26
|
+
from reconcile.utils.secret_reader import SecretReader
|
27
|
+
|
28
|
+
QONTRACT_INTEGRATION = "gcp-image-mirror"
|
29
|
+
REQUEST_TIMEOUT = 60
|
30
|
+
GCR_SECRET_PREFIX = "gcr_"
|
31
|
+
AR_SECRET_PREFIX = "ar_"
|
32
|
+
|
33
|
+
|
34
|
+
class ImageSyncItem(BaseModel):
|
35
|
+
mirror: ContainerImageMirror
|
36
|
+
destination_url: str
|
37
|
+
org_name: str
|
38
|
+
|
39
|
+
|
40
|
+
class SyncTask(BaseModel):
|
41
|
+
mirror_creds: str | None = None
|
42
|
+
source_url: str
|
43
|
+
dest_url: str
|
44
|
+
org_name: str
|
45
|
+
|
46
|
+
|
47
|
+
class QuayMirror:
|
48
|
+
def __init__(self, dry_run: bool = False) -> None:
|
49
|
+
self.dry_run = dry_run
|
50
|
+
self.gqlapi = gql.get_api()
|
51
|
+
settings = queries.get_app_interface_settings()
|
52
|
+
self.secret_reader = SecretReader(settings=settings)
|
53
|
+
self.skopeo_cli = Skopeo(dry_run)
|
54
|
+
self.push_creds = self._get_push_creds()
|
55
|
+
self.session = requests.Session()
|
56
|
+
|
57
|
+
def __enter__(self) -> Self:
|
58
|
+
return self
|
59
|
+
|
60
|
+
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
|
61
|
+
self.session.close()
|
62
|
+
|
63
|
+
def run(self) -> None:
|
64
|
+
gql_result = gql_gcp_repos.query(query_func=self.gqlapi.query)
|
65
|
+
processed_repos = self.process_repos_to_sync(gql_result)
|
66
|
+
sync_tasks = self.process_sync_tasks(processed_repos)
|
67
|
+
|
68
|
+
for task in sync_tasks:
|
69
|
+
try:
|
70
|
+
dest_creds = self.push_creds[f"{GCR_SECRET_PREFIX}{task.org_name}"]
|
71
|
+
if "pkg.dev" in task.dest_url:
|
72
|
+
dest_creds = self.push_creds[f"{AR_SECRET_PREFIX}{task.org_name}"]
|
73
|
+
|
74
|
+
self.skopeo_cli.copy(
|
75
|
+
src_image=task.source_url,
|
76
|
+
src_creds=task.mirror_creds,
|
77
|
+
dst_image=task.dest_url,
|
78
|
+
dest_creds=dest_creds,
|
79
|
+
)
|
80
|
+
except SkopeoCmdError as details:
|
81
|
+
logging.error("[%s]", details)
|
82
|
+
|
83
|
+
# processes the GQL repos to come up with a list of items that need to be synced
|
84
|
+
def process_repos_to_sync(
|
85
|
+
self, repos: gql_gcp_repos.GcpDockerReposQueryData
|
86
|
+
) -> list[ImageSyncItem]:
|
87
|
+
summary = list[ImageSyncItem]()
|
88
|
+
if repos.apps:
|
89
|
+
for app in repos.apps:
|
90
|
+
if app.gcr_repos:
|
91
|
+
for gcr_project in app.gcr_repos:
|
92
|
+
for gcr_repo in gcr_project.items:
|
93
|
+
if gcr_repo.mirror:
|
94
|
+
project_name = gcr_project.project.name
|
95
|
+
summary.append(
|
96
|
+
ImageSyncItem(
|
97
|
+
mirror=gcr_repo.mirror,
|
98
|
+
destination_url=f"gcr.io/{project_name}/{gcr_repo.name}",
|
99
|
+
org_name=project_name,
|
100
|
+
)
|
101
|
+
)
|
102
|
+
if app.artifact_registry_mirrors:
|
103
|
+
for ar_project in app.artifact_registry_mirrors:
|
104
|
+
for ar_repo in ar_project.items:
|
105
|
+
summary.append(
|
106
|
+
ImageSyncItem(
|
107
|
+
mirror=ar_repo.mirror,
|
108
|
+
destination_url=ar_repo.image_url,
|
109
|
+
org_name=ar_project.project.name,
|
110
|
+
)
|
111
|
+
)
|
112
|
+
|
113
|
+
return summary
|
114
|
+
|
115
|
+
@staticmethod
|
116
|
+
def sync_tag(
|
117
|
+
tags: list[str] | None, tags_exclude: list[str] | None, candidate: str
|
118
|
+
) -> bool:
|
119
|
+
if tags is not None:
|
120
|
+
# When tags is defined, we don't look at tags_exclude
|
121
|
+
return any(re.match(tag, candidate) for tag in tags)
|
122
|
+
|
123
|
+
if tags_exclude is not None:
|
124
|
+
return any(re.match(tag, candidate) for tag in tags_exclude)
|
125
|
+
for tag_exclude in tags_exclude:
|
126
|
+
if re.match(tag_exclude, candidate):
|
127
|
+
return False
|
128
|
+
return True
|
129
|
+
|
130
|
+
# Both tags and tags_exclude are None, so
|
131
|
+
# tag must be synced
|
132
|
+
return True
|
133
|
+
|
134
|
+
# second layer of processing that matches up pull/push creds with each repo and determines what tags need to be synced
|
135
|
+
def process_sync_tasks(self, repos_to_sync: list[ImageSyncItem]) -> list[SyncTask]:
|
136
|
+
eight_hours = 28800 # 60 * 60 * 8
|
137
|
+
is_deep_sync = self._is_deep_sync(interval=eight_hours)
|
138
|
+
|
139
|
+
sync_tasks = list[SyncTask]()
|
140
|
+
for item in repos_to_sync:
|
141
|
+
image = Image(
|
142
|
+
f"{item.destination_url}",
|
143
|
+
session=self.session,
|
144
|
+
timeout=REQUEST_TIMEOUT,
|
145
|
+
)
|
146
|
+
|
147
|
+
mirror_url = item.mirror.url
|
148
|
+
|
149
|
+
username = None
|
150
|
+
password = None
|
151
|
+
mirror_creds = None
|
152
|
+
pull_credentials = item.mirror.pull_credentials
|
153
|
+
if pull_credentials:
|
154
|
+
raw_data = self.secret_reader.read_all(pull_credentials.dict())
|
155
|
+
username = raw_data["user"]
|
156
|
+
password = raw_data["token"]
|
157
|
+
mirror_creds = f"{username}:{password}"
|
158
|
+
|
159
|
+
image_mirror = Image(
|
160
|
+
mirror_url,
|
161
|
+
username=username,
|
162
|
+
password=password,
|
163
|
+
session=self.session,
|
164
|
+
timeout=REQUEST_TIMEOUT,
|
165
|
+
)
|
166
|
+
|
167
|
+
for tag in image_mirror:
|
168
|
+
if not self.sync_tag(
|
169
|
+
tags=item.mirror.tags,
|
170
|
+
tags_exclude=item.mirror.tags_exclude,
|
171
|
+
candidate=tag,
|
172
|
+
):
|
173
|
+
continue
|
174
|
+
|
175
|
+
# the Image class allows you to fetch Image information at a specific tag with a get operator
|
176
|
+
upstream = image_mirror[tag]
|
177
|
+
downstream = image[tag]
|
178
|
+
if tag not in image:
|
179
|
+
logging.debug(
|
180
|
+
f"Image {image.image}: {downstream} and mirror {upstream} are out of sync"
|
181
|
+
)
|
182
|
+
sync_tasks.append(
|
183
|
+
SyncTask(
|
184
|
+
source_url=str(upstream),
|
185
|
+
mirror_creds=mirror_creds,
|
186
|
+
dest_url=str(downstream),
|
187
|
+
org_name=item.org_name,
|
188
|
+
)
|
189
|
+
)
|
190
|
+
continue
|
191
|
+
|
192
|
+
# Deep (slow) check only in non dry-run mode
|
193
|
+
if self.dry_run:
|
194
|
+
logging.debug(
|
195
|
+
f"Image {image.image}: {downstream} and mirror {upstream} are in sync"
|
196
|
+
)
|
197
|
+
continue
|
198
|
+
|
199
|
+
# Deep (slow) check only from time to time
|
200
|
+
if not is_deep_sync:
|
201
|
+
logging.debug(
|
202
|
+
f"Image {image.image}: {downstream} and mirror {upstream} are in sync"
|
203
|
+
)
|
204
|
+
continue
|
205
|
+
|
206
|
+
try:
|
207
|
+
if downstream == upstream:
|
208
|
+
logging.debug(
|
209
|
+
f"Image {image.image}: {downstream} and mirror {upstream} are in sync",
|
210
|
+
)
|
211
|
+
continue
|
212
|
+
except ImageComparisonError as details:
|
213
|
+
logging.error("[%s]", details)
|
214
|
+
continue
|
215
|
+
|
216
|
+
logging.debug(
|
217
|
+
f"Image {image.image}: {downstream} and mirror {upstream} are out of sync"
|
218
|
+
)
|
219
|
+
sync_tasks.append(
|
220
|
+
SyncTask(
|
221
|
+
source_url=str(upstream),
|
222
|
+
mirror_creds=mirror_creds,
|
223
|
+
dest_url=str(downstream),
|
224
|
+
org_name=item.org_name,
|
225
|
+
)
|
226
|
+
)
|
227
|
+
|
228
|
+
return sync_tasks
|
229
|
+
|
230
|
+
def _is_deep_sync(self, interval: int) -> bool:
|
231
|
+
control_file_name = "qontract-reconcile-gcp-image-mirror.timestamp"
|
232
|
+
control_file_path = os.path.join(tempfile.gettempdir(), control_file_name)
|
233
|
+
try:
|
234
|
+
with open(control_file_path, encoding="locale") as file_obj:
|
235
|
+
last_deep_sync = float(file_obj.read())
|
236
|
+
except FileNotFoundError:
|
237
|
+
self._record_timestamp(control_file_path)
|
238
|
+
return True
|
239
|
+
|
240
|
+
next_deep_sync = last_deep_sync + interval
|
241
|
+
if time.time() >= next_deep_sync:
|
242
|
+
self._record_timestamp(control_file_path)
|
243
|
+
return True
|
244
|
+
|
245
|
+
return False
|
246
|
+
|
247
|
+
def _decode_push_secret(self, secret: VaultSecret) -> str:
|
248
|
+
raw_data = self.secret_reader.read_all(secret.dict())
|
249
|
+
token = base64.b64decode(raw_data["token"]).decode()
|
250
|
+
return f"{raw_data['user']}:{token}"
|
251
|
+
|
252
|
+
@staticmethod
|
253
|
+
def _record_timestamp(path: str) -> None:
|
254
|
+
with open(path, "w", encoding="locale") as file_object:
|
255
|
+
file_object.write(str(time.time()))
|
256
|
+
|
257
|
+
def _get_push_creds(self) -> dict[str, str]:
|
258
|
+
result = gql_gcp_projects.query(query_func=self.gqlapi.query)
|
259
|
+
|
260
|
+
creds = dict[str, str]()
|
261
|
+
if result.gcp_projects:
|
262
|
+
for project_data in result.gcp_projects:
|
263
|
+
# support old pull secret for backwards compatibility (although they are both using artifact registry on the backend)
|
264
|
+
if project_data.gcr_push_credentials:
|
265
|
+
creds[f"{GCR_SECRET_PREFIX}{project_data.name}"] = (
|
266
|
+
self._decode_push_secret(project_data.gcr_push_credentials)
|
267
|
+
)
|
268
|
+
creds[f"{AR_SECRET_PREFIX}{project_data.name}"] = (
|
269
|
+
self._decode_push_secret(project_data.artifact_push_credentials)
|
270
|
+
)
|
271
|
+
return creds
|
272
|
+
|
273
|
+
|
274
|
+
def run(dry_run: bool) -> None:
|
275
|
+
with QuayMirror(dry_run) as gcp_image_mirror:
|
276
|
+
gcp_image_mirror.run()
|
File without changes
|
@@ -0,0 +1,158 @@
|
|
1
|
+
"""
|
2
|
+
Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
|
3
|
+
"""
|
4
|
+
from collections.abc import Callable # noqa: F401 # pylint: disable=W0611
|
5
|
+
from datetime import datetime # noqa: F401 # pylint: disable=W0611
|
6
|
+
from enum import Enum # noqa: F401 # pylint: disable=W0611
|
7
|
+
from typing import ( # noqa: F401 # pylint: disable=W0611
|
8
|
+
Any,
|
9
|
+
Optional,
|
10
|
+
Union,
|
11
|
+
)
|
12
|
+
|
13
|
+
from pydantic import ( # noqa: F401 # pylint: disable=W0611
|
14
|
+
BaseModel,
|
15
|
+
Extra,
|
16
|
+
Field,
|
17
|
+
Json,
|
18
|
+
)
|
19
|
+
|
20
|
+
|
21
|
+
DEFINITION = """
|
22
|
+
query AWSAccountsCloudwatchLogRetentionCleanup {
|
23
|
+
accounts: awsaccounts_v1
|
24
|
+
{
|
25
|
+
path
|
26
|
+
name
|
27
|
+
uid
|
28
|
+
terraformUsername
|
29
|
+
consoleUrl
|
30
|
+
resourcesDefaultRegion
|
31
|
+
supportedDeploymentRegions
|
32
|
+
providerVersion
|
33
|
+
accountOwners {
|
34
|
+
name
|
35
|
+
email
|
36
|
+
}
|
37
|
+
automationToken {
|
38
|
+
path
|
39
|
+
field
|
40
|
+
version
|
41
|
+
format
|
42
|
+
}
|
43
|
+
garbageCollection
|
44
|
+
enableDeletion
|
45
|
+
deletionApprovals {
|
46
|
+
type
|
47
|
+
name
|
48
|
+
expiration
|
49
|
+
}
|
50
|
+
disable {
|
51
|
+
integrations
|
52
|
+
}
|
53
|
+
deleteKeys
|
54
|
+
premiumSupport
|
55
|
+
ecrs {
|
56
|
+
region
|
57
|
+
}
|
58
|
+
partition
|
59
|
+
cleanup {
|
60
|
+
provider
|
61
|
+
... on AWSAccountCleanupOptionCloudWatch_v1 {
|
62
|
+
regex
|
63
|
+
retention_in_days
|
64
|
+
delete_empty_log_group
|
65
|
+
region
|
66
|
+
}
|
67
|
+
}
|
68
|
+
}
|
69
|
+
}
|
70
|
+
"""
|
71
|
+
|
72
|
+
|
73
|
+
class ConfiguredBaseModel(BaseModel):
|
74
|
+
class Config:
|
75
|
+
smart_union=True
|
76
|
+
extra=Extra.forbid
|
77
|
+
|
78
|
+
|
79
|
+
class OwnerV1(ConfiguredBaseModel):
|
80
|
+
name: str = Field(..., alias="name")
|
81
|
+
email: str = Field(..., alias="email")
|
82
|
+
|
83
|
+
|
84
|
+
class VaultSecretV1(ConfiguredBaseModel):
|
85
|
+
path: str = Field(..., alias="path")
|
86
|
+
field: str = Field(..., alias="field")
|
87
|
+
version: Optional[int] = Field(..., alias="version")
|
88
|
+
q_format: Optional[str] = Field(..., alias="format")
|
89
|
+
|
90
|
+
|
91
|
+
class DeletionApprovalV1(ConfiguredBaseModel):
|
92
|
+
q_type: str = Field(..., alias="type")
|
93
|
+
name: str = Field(..., alias="name")
|
94
|
+
expiration: str = Field(..., alias="expiration")
|
95
|
+
|
96
|
+
|
97
|
+
class DisableClusterAutomationsV1(ConfiguredBaseModel):
|
98
|
+
integrations: Optional[list[str]] = Field(..., alias="integrations")
|
99
|
+
|
100
|
+
|
101
|
+
class AWSECRV1(ConfiguredBaseModel):
|
102
|
+
region: str = Field(..., alias="region")
|
103
|
+
|
104
|
+
|
105
|
+
class AWSAccountCleanupOptionV1(ConfiguredBaseModel):
|
106
|
+
provider: str = Field(..., alias="provider")
|
107
|
+
|
108
|
+
|
109
|
+
class AWSAccountCleanupOptionCloudWatchV1(AWSAccountCleanupOptionV1):
|
110
|
+
regex: str = Field(..., alias="regex")
|
111
|
+
retention_in_days: int = Field(..., alias="retention_in_days")
|
112
|
+
delete_empty_log_group: Optional[bool] = Field(..., alias="delete_empty_log_group")
|
113
|
+
region: Optional[str] = Field(..., alias="region")
|
114
|
+
|
115
|
+
|
116
|
+
class AWSAccountV1(ConfiguredBaseModel):
|
117
|
+
path: str = Field(..., alias="path")
|
118
|
+
name: str = Field(..., alias="name")
|
119
|
+
uid: str = Field(..., alias="uid")
|
120
|
+
terraform_username: Optional[str] = Field(..., alias="terraformUsername")
|
121
|
+
console_url: str = Field(..., alias="consoleUrl")
|
122
|
+
resources_default_region: str = Field(..., alias="resourcesDefaultRegion")
|
123
|
+
supported_deployment_regions: Optional[list[str]] = Field(..., alias="supportedDeploymentRegions")
|
124
|
+
provider_version: str = Field(..., alias="providerVersion")
|
125
|
+
account_owners: list[OwnerV1] = Field(..., alias="accountOwners")
|
126
|
+
automation_token: VaultSecretV1 = Field(..., alias="automationToken")
|
127
|
+
garbage_collection: Optional[bool] = Field(..., alias="garbageCollection")
|
128
|
+
enable_deletion: Optional[bool] = Field(..., alias="enableDeletion")
|
129
|
+
deletion_approvals: Optional[list[DeletionApprovalV1]] = Field(..., alias="deletionApprovals")
|
130
|
+
disable: Optional[DisableClusterAutomationsV1] = Field(..., alias="disable")
|
131
|
+
delete_keys: Optional[list[str]] = Field(..., alias="deleteKeys")
|
132
|
+
premium_support: bool = Field(..., alias="premiumSupport")
|
133
|
+
ecrs: Optional[list[AWSECRV1]] = Field(..., alias="ecrs")
|
134
|
+
partition: Optional[str] = Field(..., alias="partition")
|
135
|
+
cleanup: Optional[list[Union[AWSAccountCleanupOptionCloudWatchV1, AWSAccountCleanupOptionV1]]] = Field(..., alias="cleanup")
|
136
|
+
|
137
|
+
|
138
|
+
class AWSAccountsCloudwatchLogRetentionCleanupQueryData(ConfiguredBaseModel):
|
139
|
+
accounts: Optional[list[AWSAccountV1]] = Field(..., alias="accounts")
|
140
|
+
|
141
|
+
|
142
|
+
def query(query_func: Callable, **kwargs: Any) -> AWSAccountsCloudwatchLogRetentionCleanupQueryData:
|
143
|
+
"""
|
144
|
+
This is a convenience function which queries and parses the data into
|
145
|
+
concrete types. It should be compatible with most GQL clients.
|
146
|
+
You do not have to use it to consume the generated data classes.
|
147
|
+
Alternatively, you can also mime and alternate the behavior
|
148
|
+
of this function in the caller.
|
149
|
+
|
150
|
+
Parameters:
|
151
|
+
query_func (Callable): Function which queries your GQL Server
|
152
|
+
kwargs: optional arguments that will be passed to the query function
|
153
|
+
|
154
|
+
Returns:
|
155
|
+
AWSAccountsCloudwatchLogRetentionCleanupQueryData: queried data parsed into generated classes
|
156
|
+
"""
|
157
|
+
raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
|
158
|
+
return AWSAccountsCloudwatchLogRetentionCleanupQueryData(**raw_data)
|