qontract-reconcile 0.10.2.dev159__py3-none-any.whl → 0.10.2.dev161__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: qontract-reconcile
3
- Version: 0.10.2.dev159
3
+ Version: 0.10.2.dev161
4
4
  Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
5
5
  Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
6
6
  Project-URL: repository, https://github.com/app-sre/qontract-reconcile
@@ -10,7 +10,7 @@ reconcile/aws_iam_password_reset.py,sha256=O0JX2N5kNRKs3u2xzu4NNrI6p0ag5JWy3MTsv
10
10
  reconcile/aws_support_cases_sos.py,sha256=PDhilxQ4TBxVnxUPIUdTbKEaNUI0wzPiEsB91oHT2fY,3384
11
11
  reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=O1wFp52EyF538c6txaWBs8eMtUIy19gyHZ6VzJ6QXS8,3512
12
12
  reconcile/checkpoint.py,sha256=_JhMxrye5BgkRMxWYuf7Upli6XayPINKSsuo3ynHTRc,5010
13
- reconcile/cli.py,sha256=hwqPcZVmazrhzq1esPBk5jNHSzpfr7o9EmuuMqiPxfg,108430
13
+ reconcile/cli.py,sha256=xyVnxNyq3IPISWwFlB9j4HAFjowXYv3EdsEGIMFhTy0,108438
14
14
  reconcile/closedbox_endpoint_monitoring_base.py,sha256=al7m8EgnnYx90rY1REryW3byN_ItfJfAzEeLtjbCfi0,4921
15
15
  reconcile/cluster_deployment_mapper.py,sha256=5gumAaRCcFXsabUJ1dnuUy9WrP_FEEM5JnOnE8ch9sE,2326
16
16
  reconcile/dashdotdb_base.py,sha256=83ZWIf5JJk3P_D69y2TmXRcQr6ELJGlv10OM0h7fJVs,4767
@@ -22,7 +22,7 @@ reconcile/database_access_manager.py,sha256=Z3aAmw2LsmMIIor-bOGzziVZdVNC82Gmw8oH
22
22
  reconcile/deadmanssnitch.py,sha256=n-5W-djUgwzpmdDM4eQIZpkkDmHY0vndt-42LJXI4Y8,7491
23
23
  reconcile/email_sender.py,sha256=38Wvl6WHqCwlqLx4oxVJOIeDmoJsyitD3g1F4jTkAj8,4246
24
24
  reconcile/gabi_authorized_users.py,sha256=Jwvo97nzUX3NIl2VHKuZlT0-I40qk2VnACbafe91T2o,4854
25
- reconcile/gcr_mirror.py,sha256=cdTd0CZU0qUsXJqe5k4dgpMQlyk__nyeGH0f_Cky3C0,8957
25
+ reconcile/gcp_image_mirror.py,sha256=M5pimd0j13BBWCa8vX0fftWO0pHBfQCATIIpOGggDSA,10332
26
26
  reconcile/github_org.py,sha256=Wc5cZamatuWsW2ZJT2ib5ps8l3iY3RXHwNUxVJerqz0,14173
27
27
  reconcile/github_owners.py,sha256=viE1KJ-zaTxuZ5yItg2C263J0brn-Q-3hR_DkYDMbhY,3122
28
28
  reconcile/github_repo_invites.py,sha256=U9UCzNVwrZ7MqODtFah8ogH0NNY-XjBin7G9gqHtCUY,2690
@@ -152,7 +152,7 @@ reconcile/aws_account_manager/utils.py,sha256=iYPPOtbZ7FiKkz9v5f1YXRIHw5YFOtSavU
152
152
  reconcile/aws_ami_cleanup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
153
153
  reconcile/aws_ami_cleanup/integration.py,sha256=KG7g9NpbKmoaveDD3oi9SinqUE29NaM-4lGo-6YuHlM,9302
154
154
  reconcile/aws_cloudwatch_log_retention/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
155
- reconcile/aws_cloudwatch_log_retention/integration.py,sha256=qVggTjGwP_6xTCXo2o3brSPtjwArNhZiGPy8YiBrCMM,7779
155
+ reconcile/aws_cloudwatch_log_retention/integration.py,sha256=QY5EtCpcMN0TgOQInIDW65wT1YksMBFkK8aNK5cg-XA,8107
156
156
  reconcile/aws_saml_idp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
157
157
  reconcile/aws_saml_idp/integration.py,sha256=Z2JtUx2YIbkn0KVrVa2CoAErPB8vTykOOkWD_ZPoB94,6511
158
158
  reconcile/aws_saml_roles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -227,7 +227,7 @@ reconcile/glitchtip_project_alerts/integration.py,sha256=BgMx-NyV9mTuv7Sotb2OioC
227
227
  reconcile/glitchtip_project_dsn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
228
228
  reconcile/glitchtip_project_dsn/integration.py,sha256=2iugub-kHYkHNK33n0v9_TeWonuxCPah_VkoTPvaajE,8077
229
229
  reconcile/gql_definitions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
230
- reconcile/gql_definitions/introspection.json,sha256=myUuHC_BLY3xZ0nDjnaQsvYFNM5eTiE3bWgNgM3e5iI,2290863
230
+ reconcile/gql_definitions/introspection.json,sha256=wNGZv8V6ivviCcwPLw34nzTR5fgwKRyE68Z93HbmBBs,2296576
231
231
  reconcile/gql_definitions/acs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
232
232
  reconcile/gql_definitions/acs/acs_instances.py,sha256=L91WW9LbhJbBSrECqShQpFtjoBOsmNIYLRpMbx1io5o,2181
233
233
  reconcile/gql_definitions/acs/acs_policies.py,sha256=bN5i4mks10Z23KJSj7jqp966Osq2dps4d-sPH9gjxEA,7008
@@ -246,6 +246,8 @@ reconcile/gql_definitions/aws_account_manager/__init__.py,sha256=47DEQpj8HBSa-_T
246
246
  reconcile/gql_definitions/aws_account_manager/aws_accounts.py,sha256=vF51KrY2gwX0J9vESiaRMPQqdAMEtz9f_tBq52bInp0,5148
247
247
  reconcile/gql_definitions/aws_ami_cleanup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
248
248
  reconcile/gql_definitions/aws_ami_cleanup/aws_accounts.py,sha256=jIgOa888MYLLvVsn1ir3nbkhWLG5T6dBg7oDnp1q8BI,4108
249
+ reconcile/gql_definitions/aws_cloudwatch_log_retention/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
250
+ reconcile/gql_definitions/aws_cloudwatch_log_retention/aws_accounts.py,sha256=Am6y5LZIBncH9u7vwU48WRksnwGljpoS-xbP7MJA8-4,4976
249
251
  reconcile/gql_definitions/aws_saml_idp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
250
252
  reconcile/gql_definitions/aws_saml_idp/aws_accounts.py,sha256=pR9Qm6P9Roe4OJaDXvfm8AcfkSSAtriQdlwLwW7UdUU,2666
251
253
  reconcile/gql_definitions/aws_saml_roles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -333,6 +335,7 @@ reconcile/gql_definitions/fragments/aws_infra_management_account.py,sha256=uAmAL
333
335
  reconcile/gql_definitions/fragments/aws_vpc.py,sha256=T2egTwi2Rb0IRBBmsyag8xKpu_m6GbIAy80fhZNZwk8,1434
334
336
  reconcile/gql_definitions/fragments/aws_vpc_request.py,sha256=o0qUsPrFXs8GAbtgMXQmIJxc1mw5skSIzCcidE857g8,2460
335
337
  reconcile/gql_definitions/fragments/aws_vpc_request_subnet.py,sha256=qaTFT8cGzEslw51nUeb45Nfnv6kFxUm4CWrRR3xfBvA,760
338
+ reconcile/gql_definitions/fragments/container_image_mirror.py,sha256=qyfQlnKUCzFEPgUJ9VGmDYFmiGHR7VZ_YJNd4KeoolM,968
336
339
  reconcile/gql_definitions/fragments/deplopy_resources.py,sha256=0u3xYqL5NpMf149BJLfPhHqAOWu06aLULdNk_2Mulxg,1089
337
340
  reconcile/gql_definitions/fragments/disable.py,sha256=Ojw98OSxcovrtmw_aAyhaVHhIa1MSUbBfKX4i2IpI74,715
338
341
  reconcile/gql_definitions/fragments/email_service.py,sha256=0wKpICsg4pcMfr2lszvnqbuPX7wVYoJ5cYFU2uQkHbY,803
@@ -353,6 +356,9 @@ reconcile/gql_definitions/fragments/terraform_state.py,sha256=S5QuTR9YlvUObiU7he
353
356
  reconcile/gql_definitions/fragments/upgrade_policy.py,sha256=cVza8zfra1E3yBsHiS-hKbys17fvv572GFnKshJjluE,1246
354
357
  reconcile/gql_definitions/fragments/user.py,sha256=TZyFEs1fBg5PkvWdyCxFDZ_3aRhcQzusfhObXFiOU_0,1025
355
358
  reconcile/gql_definitions/fragments/vault_secret.py,sha256=8xoQJNx1jKw_1yradq1iLEYWzuOHra1bEHHU7WHKxqo,833
359
+ reconcile/gql_definitions/gcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
360
+ reconcile/gql_definitions/gcp/gcp_docker_repos.py,sha256=HvNaJxQYNPBTDmk26cOUY5_C5oBfau4bdfuI-L1Vcps,3338
361
+ reconcile/gql_definitions/gcp/gcp_projects.py,sha256=7LslYFSN2r8vYhYdi4s-zOkIrqpqyt4ayxbPNMie9M4,2108
356
362
  reconcile/gql_definitions/gitlab_members/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
357
363
  reconcile/gql_definitions/gitlab_members/gitlab_instances.py,sha256=oYPvfiOsPTGHXQeSfxXvBuvJFrwp0VtE2F0lVKFQMoU,2206
358
364
  reconcile/gql_definitions/gitlab_members/permissions.py,sha256=Qzj3Fpv7xj8v9eygeP312nHRNg8er8XMRBveynPIyQM,3302
@@ -575,6 +581,7 @@ reconcile/typed_queries/vault.py,sha256=lkRsmobykorof3fcrIPLz-NgvAiSOWSOZc_jXBln
575
581
  reconcile/typed_queries/app_interface_metrics_exporter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
576
582
  reconcile/typed_queries/app_interface_metrics_exporter/onboarding_status.py,sha256=X-N1WJGOL6OR9940P0_K4-YJzkL5Vg4favhYrBxXD9A,327
577
583
  reconcile/typed_queries/app_interface_metrics_exporter/terraform_repo.py,sha256=r-nJ5CucAOE_cwxnbVp5lmAAfHBG8t1h2tVmhviVYls,290
584
+ reconcile/typed_queries/aws_cloudwatch_log_retention/aws_accounts.py,sha256=WiQ84vEZp-oYvD4CVZaFDYcMBn-pkO3slwsHIQxvHks,296
578
585
  reconcile/typed_queries/cost_report/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
579
586
  reconcile/typed_queries/cost_report/app_names.py,sha256=HMEMIqAbMyVQfoQ5YXTXE4xDt7FaXBRz0QIHnsIZC1c,478
580
587
  reconcile/typed_queries/cost_report/cost_namespaces.py,sha256=1GUjWXQj7U2djVHBPYcd8Cy2-enKXf0-GaplLi8JZw4,1178
@@ -797,7 +804,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
797
804
  tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
798
805
  tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
799
806
  tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
800
- qontract_reconcile-0.10.2.dev159.dist-info/METADATA,sha256=xiCIKDXtULyvmvHhV2FclCguppmcLFNhWWsIwmKbsyU,24627
801
- qontract_reconcile-0.10.2.dev159.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
802
- qontract_reconcile-0.10.2.dev159.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
803
- qontract_reconcile-0.10.2.dev159.dist-info/RECORD,,
807
+ qontract_reconcile-0.10.2.dev161.dist-info/METADATA,sha256=8F0IJKYcSCyZq47ZOd2Bn6zFBeo2I83momD5TwvCTSg,24627
808
+ qontract_reconcile-0.10.2.dev161.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
809
+ qontract_reconcile-0.10.2.dev161.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
810
+ qontract_reconcile-0.10.2.dev161.dist-info/RECORD,,
@@ -5,13 +5,22 @@ from collections import defaultdict
5
5
  from collections.abc import Iterable
6
6
  from datetime import UTC, datetime, timedelta
7
7
  from enum import Enum
8
- from typing import TYPE_CHECKING
8
+ from typing import (
9
+ TYPE_CHECKING,
10
+ )
9
11
 
10
12
  from botocore.exceptions import ClientError
11
13
  from pydantic import BaseModel
12
14
 
13
15
  from reconcile import queries
14
- from reconcile.queries import get_aws_accounts
16
+ from reconcile.gql_definitions.aws_cloudwatch_log_retention.aws_accounts import (
17
+ AWSAccountCleanupOptionCloudWatchV1,
18
+ AWSAccountV1,
19
+ )
20
+ from reconcile.typed_queries.aws_cloudwatch_log_retention.aws_accounts import (
21
+ get_aws_accounts,
22
+ )
23
+ from reconcile.utils import gql
15
24
  from reconcile.utils.aws_api import AWSApi
16
25
 
17
26
  if TYPE_CHECKING:
@@ -39,31 +48,33 @@ DEFAULT_AWS_CLOUDWATCH_CLEANUP_OPTION = AWSCloudwatchCleanupOption(
39
48
 
40
49
 
41
50
  def get_desired_cleanup_options_by_region(
42
- account: dict,
51
+ account: AWSAccountV1,
43
52
  ) -> dict[str, list[AWSCloudwatchCleanupOption]]:
44
- default_region = account["resourcesDefaultRegion"]
53
+ default_region = account.resources_default_region
45
54
  result = defaultdict(list)
46
- if cleanup := account.get("cleanup"):
47
- for cleanup_option in cleanup:
48
- if cleanup_option["provider"] == "cloudwatch":
49
- region = cleanup_option.get("region") or default_region
50
- result[region].append(
51
- AWSCloudwatchCleanupOption(
52
- regex=re.compile(cleanup_option["regex"]),
53
- retention_in_days=cleanup_option["retention_in_days"],
54
- delete_empty_log_group=bool(
55
- cleanup_option["delete_empty_log_group"]
56
- ),
57
- )
55
+ for cleanup_option in account.cleanup or []:
56
+ if isinstance(cleanup_option, AWSAccountCleanupOptionCloudWatchV1):
57
+ region = cleanup_option.region or default_region
58
+ result[region].append(
59
+ AWSCloudwatchCleanupOption(
60
+ regex=re.compile(cleanup_option.regex),
61
+ retention_in_days=cleanup_option.retention_in_days,
62
+ delete_empty_log_group=bool(cleanup_option.delete_empty_log_group),
58
63
  )
64
+ )
59
65
  if not result:
60
66
  result[default_region].append(DEFAULT_AWS_CLOUDWATCH_CLEANUP_OPTION)
61
67
  return result
62
68
 
63
69
 
64
- def create_awsapi_client(accounts: list, thread_pool_size: int) -> AWSApi:
70
+ def create_awsapi_client(accounts: list[AWSAccountV1], thread_pool_size: int) -> AWSApi:
65
71
  settings = queries.get_secret_reader_settings()
66
- return AWSApi(thread_pool_size, accounts, settings=settings, init_users=False)
72
+ return AWSApi(
73
+ thread_pool_size,
74
+ [account.dict(by_alias=True) for account in accounts],
75
+ settings=settings,
76
+ init_users=False,
77
+ )
67
78
 
68
79
 
69
80
  def is_empty(log_group: LogGroupTypeDef) -> bool:
@@ -192,10 +203,10 @@ def _find_desired_cleanup_option(
192
203
 
193
204
  def _reconcile_log_groups(
194
205
  dry_run: bool,
195
- aws_account: dict,
206
+ aws_account: AWSAccountV1,
196
207
  awsapi: AWSApi,
197
208
  ) -> None:
198
- account_name = aws_account["name"]
209
+ account_name = aws_account.name
199
210
  desired_cleanup_options_by_region = get_desired_cleanup_options_by_region(
200
211
  aws_account
201
212
  )
@@ -230,12 +241,15 @@ def _reconcile_log_groups(
230
241
  )
231
242
 
232
243
 
233
- def get_active_aws_accounts() -> list[dict]:
244
+ def get_active_aws_accounts() -> list[AWSAccountV1]:
234
245
  return [
235
- a
236
- for a in get_aws_accounts(cleanup=True)
237
- if "aws-cloudwatch-log-retention"
238
- not in (a.get("disable") or {}).get("integrations", [])
246
+ account
247
+ for account in get_aws_accounts(gql.get_api())
248
+ if not (
249
+ account.disable
250
+ and account.disable.integrations
251
+ and "aws-cloudwatch-log-retention" in account.disable.integrations
252
+ )
239
253
  ]
240
254
 
241
255
 
reconcile/cli.py CHANGED
@@ -1849,15 +1849,13 @@ def quay_membership(ctx):
1849
1849
  run_integration(reconcile.quay_membership, ctx.obj)
1850
1850
 
1851
1851
 
1852
- @integration.command(
1853
- short_help="Mirrors external images into Google Container Registry."
1854
- )
1852
+ @integration.command(short_help="Mirrors external images into GCP Artifact Registry.")
1855
1853
  @click.pass_context
1856
1854
  @binary(["skopeo"])
1857
- def gcr_mirror(ctx):
1858
- import reconcile.gcr_mirror
1855
+ def gcp_image_mirror(ctx):
1856
+ import reconcile.gcp_image_mirror
1859
1857
 
1860
- run_integration(reconcile.gcr_mirror, ctx.obj)
1858
+ run_integration(reconcile.gcp_image_mirror, ctx.obj)
1861
1859
 
1862
1860
 
1863
1861
  @integration.command(short_help="Mirrors external images into Quay.")
@@ -0,0 +1,276 @@
1
+ import base64
2
+ import logging
3
+ import os
4
+ import re
5
+ import tempfile
6
+ import time
7
+ from typing import Any, Self
8
+
9
+ import requests
10
+ from pydantic import BaseModel
11
+ from sretoolbox.container import (
12
+ Image,
13
+ Skopeo,
14
+ )
15
+ from sretoolbox.container.image import ImageComparisonError
16
+ from sretoolbox.container.skopeo import SkopeoCmdError
17
+
18
+ import reconcile.gql_definitions.gcp.gcp_docker_repos as gql_gcp_repos
19
+ import reconcile.gql_definitions.gcp.gcp_projects as gql_gcp_projects
20
+ from reconcile import queries
21
+ from reconcile.gql_definitions.fragments.container_image_mirror import (
22
+ ContainerImageMirror,
23
+ )
24
+ from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
25
+ from reconcile.utils import gql
26
+ from reconcile.utils.secret_reader import SecretReader
27
+
28
+ QONTRACT_INTEGRATION = "gcp-image-mirror"
29
+ REQUEST_TIMEOUT = 60
30
+ GCR_SECRET_PREFIX = "gcr_"
31
+ AR_SECRET_PREFIX = "ar_"
32
+
33
+
34
+ class ImageSyncItem(BaseModel):
35
+ mirror: ContainerImageMirror
36
+ destination_url: str
37
+ org_name: str
38
+
39
+
40
+ class SyncTask(BaseModel):
41
+ mirror_creds: str | None = None
42
+ source_url: str
43
+ dest_url: str
44
+ org_name: str
45
+
46
+
47
+ class QuayMirror:
48
+ def __init__(self, dry_run: bool = False) -> None:
49
+ self.dry_run = dry_run
50
+ self.gqlapi = gql.get_api()
51
+ settings = queries.get_app_interface_settings()
52
+ self.secret_reader = SecretReader(settings=settings)
53
+ self.skopeo_cli = Skopeo(dry_run)
54
+ self.push_creds = self._get_push_creds()
55
+ self.session = requests.Session()
56
+
57
+ def __enter__(self) -> Self:
58
+ return self
59
+
60
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
61
+ self.session.close()
62
+
63
+ def run(self) -> None:
64
+ gql_result = gql_gcp_repos.query(query_func=self.gqlapi.query)
65
+ processed_repos = self.process_repos_to_sync(gql_result)
66
+ sync_tasks = self.process_sync_tasks(processed_repos)
67
+
68
+ for task in sync_tasks:
69
+ try:
70
+ dest_creds = self.push_creds[f"{GCR_SECRET_PREFIX}{task.org_name}"]
71
+ if "pkg.dev" in task.dest_url:
72
+ dest_creds = self.push_creds[f"{AR_SECRET_PREFIX}{task.org_name}"]
73
+
74
+ self.skopeo_cli.copy(
75
+ src_image=task.source_url,
76
+ src_creds=task.mirror_creds,
77
+ dst_image=task.dest_url,
78
+ dest_creds=dest_creds,
79
+ )
80
+ except SkopeoCmdError as details:
81
+ logging.error("[%s]", details)
82
+
83
+ # processes the GQL repos to come up with a list of items that need to be synced
84
+ def process_repos_to_sync(
85
+ self, repos: gql_gcp_repos.GcpDockerReposQueryData
86
+ ) -> list[ImageSyncItem]:
87
+ summary = list[ImageSyncItem]()
88
+ if repos.apps:
89
+ for app in repos.apps:
90
+ if app.gcr_repos:
91
+ for gcr_project in app.gcr_repos:
92
+ for gcr_repo in gcr_project.items:
93
+ if gcr_repo.mirror:
94
+ project_name = gcr_project.project.name
95
+ summary.append(
96
+ ImageSyncItem(
97
+ mirror=gcr_repo.mirror,
98
+ destination_url=f"gcr.io/{project_name}/{gcr_repo.name}",
99
+ org_name=project_name,
100
+ )
101
+ )
102
+ if app.artifact_registry_mirrors:
103
+ for ar_project in app.artifact_registry_mirrors:
104
+ for ar_repo in ar_project.items:
105
+ summary.append(
106
+ ImageSyncItem(
107
+ mirror=ar_repo.mirror,
108
+ destination_url=ar_repo.image_url,
109
+ org_name=ar_project.project.name,
110
+ )
111
+ )
112
+
113
+ return summary
114
+
115
+ @staticmethod
116
+ def sync_tag(
117
+ tags: list[str] | None, tags_exclude: list[str] | None, candidate: str
118
+ ) -> bool:
119
+ if tags is not None:
120
+ # When tags is defined, we don't look at tags_exclude
121
+ return any(re.match(tag, candidate) for tag in tags)
122
+
123
+ if tags_exclude is not None:
124
+ return any(re.match(tag, candidate) for tag in tags_exclude)
125
+ for tag_exclude in tags_exclude:
126
+ if re.match(tag_exclude, candidate):
127
+ return False
128
+ return True
129
+
130
+ # Both tags and tags_exclude are None, so
131
+ # tag must be synced
132
+ return True
133
+
134
+ # second layer of processing that matches up pull/push creds with each repo and determines what tags need to be synced
135
+ def process_sync_tasks(self, repos_to_sync: list[ImageSyncItem]) -> list[SyncTask]:
136
+ eight_hours = 28800 # 60 * 60 * 8
137
+ is_deep_sync = self._is_deep_sync(interval=eight_hours)
138
+
139
+ sync_tasks = list[SyncTask]()
140
+ for item in repos_to_sync:
141
+ image = Image(
142
+ f"{item.destination_url}",
143
+ session=self.session,
144
+ timeout=REQUEST_TIMEOUT,
145
+ )
146
+
147
+ mirror_url = item.mirror.url
148
+
149
+ username = None
150
+ password = None
151
+ mirror_creds = None
152
+ pull_credentials = item.mirror.pull_credentials
153
+ if pull_credentials:
154
+ raw_data = self.secret_reader.read_all(pull_credentials.dict())
155
+ username = raw_data["user"]
156
+ password = raw_data["token"]
157
+ mirror_creds = f"{username}:{password}"
158
+
159
+ image_mirror = Image(
160
+ mirror_url,
161
+ username=username,
162
+ password=password,
163
+ session=self.session,
164
+ timeout=REQUEST_TIMEOUT,
165
+ )
166
+
167
+ for tag in image_mirror:
168
+ if not self.sync_tag(
169
+ tags=item.mirror.tags,
170
+ tags_exclude=item.mirror.tags_exclude,
171
+ candidate=tag,
172
+ ):
173
+ continue
174
+
175
+ # the Image class allows you to fetch Image information at a specific tag with a get operator
176
+ upstream = image_mirror[tag]
177
+ downstream = image[tag]
178
+ if tag not in image:
179
+ logging.debug(
180
+ f"Image {image.image}: {downstream} and mirror {upstream} are out of sync"
181
+ )
182
+ sync_tasks.append(
183
+ SyncTask(
184
+ source_url=str(upstream),
185
+ mirror_creds=mirror_creds,
186
+ dest_url=str(downstream),
187
+ org_name=item.org_name,
188
+ )
189
+ )
190
+ continue
191
+
192
+ # Deep (slow) check only in non dry-run mode
193
+ if self.dry_run:
194
+ logging.debug(
195
+ f"Image {image.image}: {downstream} and mirror {upstream} are in sync"
196
+ )
197
+ continue
198
+
199
+ # Deep (slow) check only from time to time
200
+ if not is_deep_sync:
201
+ logging.debug(
202
+ f"Image {image.image}: {downstream} and mirror {upstream} are in sync"
203
+ )
204
+ continue
205
+
206
+ try:
207
+ if downstream == upstream:
208
+ logging.debug(
209
+ f"Image {image.image}: {downstream} and mirror {upstream} are in sync",
210
+ )
211
+ continue
212
+ except ImageComparisonError as details:
213
+ logging.error("[%s]", details)
214
+ continue
215
+
216
+ logging.debug(
217
+ f"Image {image.image}: {downstream} and mirror {upstream} are out of sync"
218
+ )
219
+ sync_tasks.append(
220
+ SyncTask(
221
+ source_url=str(upstream),
222
+ mirror_creds=mirror_creds,
223
+ dest_url=str(downstream),
224
+ org_name=item.org_name,
225
+ )
226
+ )
227
+
228
+ return sync_tasks
229
+
230
+ def _is_deep_sync(self, interval: int) -> bool:
231
+ control_file_name = "qontract-reconcile-gcp-image-mirror.timestamp"
232
+ control_file_path = os.path.join(tempfile.gettempdir(), control_file_name)
233
+ try:
234
+ with open(control_file_path, encoding="locale") as file_obj:
235
+ last_deep_sync = float(file_obj.read())
236
+ except FileNotFoundError:
237
+ self._record_timestamp(control_file_path)
238
+ return True
239
+
240
+ next_deep_sync = last_deep_sync + interval
241
+ if time.time() >= next_deep_sync:
242
+ self._record_timestamp(control_file_path)
243
+ return True
244
+
245
+ return False
246
+
247
+ def _decode_push_secret(self, secret: VaultSecret) -> str:
248
+ raw_data = self.secret_reader.read_all(secret.dict())
249
+ token = base64.b64decode(raw_data["token"]).decode()
250
+ return f"{raw_data['user']}:{token}"
251
+
252
+ @staticmethod
253
+ def _record_timestamp(path: str) -> None:
254
+ with open(path, "w", encoding="locale") as file_object:
255
+ file_object.write(str(time.time()))
256
+
257
+ def _get_push_creds(self) -> dict[str, str]:
258
+ result = gql_gcp_projects.query(query_func=self.gqlapi.query)
259
+
260
+ creds = dict[str, str]()
261
+ if result.gcp_projects:
262
+ for project_data in result.gcp_projects:
263
+ # support old pull secret for backwards compatibility (although they are both using artifact registry on the backend)
264
+ if project_data.gcr_push_credentials:
265
+ creds[f"{GCR_SECRET_PREFIX}{project_data.name}"] = (
266
+ self._decode_push_secret(project_data.gcr_push_credentials)
267
+ )
268
+ creds[f"{AR_SECRET_PREFIX}{project_data.name}"] = (
269
+ self._decode_push_secret(project_data.artifact_push_credentials)
270
+ )
271
+ return creds
272
+
273
+
274
+ def run(dry_run: bool) -> None:
275
+ with QuayMirror(dry_run) as gcp_image_mirror:
276
+ gcp_image_mirror.run()
@@ -0,0 +1,158 @@
1
+ """
2
+ Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
3
+ """
4
+ from collections.abc import Callable # noqa: F401 # pylint: disable=W0611
5
+ from datetime import datetime # noqa: F401 # pylint: disable=W0611
6
+ from enum import Enum # noqa: F401 # pylint: disable=W0611
7
+ from typing import ( # noqa: F401 # pylint: disable=W0611
8
+ Any,
9
+ Optional,
10
+ Union,
11
+ )
12
+
13
+ from pydantic import ( # noqa: F401 # pylint: disable=W0611
14
+ BaseModel,
15
+ Extra,
16
+ Field,
17
+ Json,
18
+ )
19
+
20
+
21
+ DEFINITION = """
22
+ query AWSAccountsCloudwatchLogRetentionCleanup {
23
+ accounts: awsaccounts_v1
24
+ {
25
+ path
26
+ name
27
+ uid
28
+ terraformUsername
29
+ consoleUrl
30
+ resourcesDefaultRegion
31
+ supportedDeploymentRegions
32
+ providerVersion
33
+ accountOwners {
34
+ name
35
+ email
36
+ }
37
+ automationToken {
38
+ path
39
+ field
40
+ version
41
+ format
42
+ }
43
+ garbageCollection
44
+ enableDeletion
45
+ deletionApprovals {
46
+ type
47
+ name
48
+ expiration
49
+ }
50
+ disable {
51
+ integrations
52
+ }
53
+ deleteKeys
54
+ premiumSupport
55
+ ecrs {
56
+ region
57
+ }
58
+ partition
59
+ cleanup {
60
+ provider
61
+ ... on AWSAccountCleanupOptionCloudWatch_v1 {
62
+ regex
63
+ retention_in_days
64
+ delete_empty_log_group
65
+ region
66
+ }
67
+ }
68
+ }
69
+ }
70
+ """
71
+
72
+
73
+ class ConfiguredBaseModel(BaseModel):
74
+ class Config:
75
+ smart_union=True
76
+ extra=Extra.forbid
77
+
78
+
79
+ class OwnerV1(ConfiguredBaseModel):
80
+ name: str = Field(..., alias="name")
81
+ email: str = Field(..., alias="email")
82
+
83
+
84
+ class VaultSecretV1(ConfiguredBaseModel):
85
+ path: str = Field(..., alias="path")
86
+ field: str = Field(..., alias="field")
87
+ version: Optional[int] = Field(..., alias="version")
88
+ q_format: Optional[str] = Field(..., alias="format")
89
+
90
+
91
+ class DeletionApprovalV1(ConfiguredBaseModel):
92
+ q_type: str = Field(..., alias="type")
93
+ name: str = Field(..., alias="name")
94
+ expiration: str = Field(..., alias="expiration")
95
+
96
+
97
+ class DisableClusterAutomationsV1(ConfiguredBaseModel):
98
+ integrations: Optional[list[str]] = Field(..., alias="integrations")
99
+
100
+
101
+ class AWSECRV1(ConfiguredBaseModel):
102
+ region: str = Field(..., alias="region")
103
+
104
+
105
+ class AWSAccountCleanupOptionV1(ConfiguredBaseModel):
106
+ provider: str = Field(..., alias="provider")
107
+
108
+
109
+ class AWSAccountCleanupOptionCloudWatchV1(AWSAccountCleanupOptionV1):
110
+ regex: str = Field(..., alias="regex")
111
+ retention_in_days: int = Field(..., alias="retention_in_days")
112
+ delete_empty_log_group: Optional[bool] = Field(..., alias="delete_empty_log_group")
113
+ region: Optional[str] = Field(..., alias="region")
114
+
115
+
116
+ class AWSAccountV1(ConfiguredBaseModel):
117
+ path: str = Field(..., alias="path")
118
+ name: str = Field(..., alias="name")
119
+ uid: str = Field(..., alias="uid")
120
+ terraform_username: Optional[str] = Field(..., alias="terraformUsername")
121
+ console_url: str = Field(..., alias="consoleUrl")
122
+ resources_default_region: str = Field(..., alias="resourcesDefaultRegion")
123
+ supported_deployment_regions: Optional[list[str]] = Field(..., alias="supportedDeploymentRegions")
124
+ provider_version: str = Field(..., alias="providerVersion")
125
+ account_owners: list[OwnerV1] = Field(..., alias="accountOwners")
126
+ automation_token: VaultSecretV1 = Field(..., alias="automationToken")
127
+ garbage_collection: Optional[bool] = Field(..., alias="garbageCollection")
128
+ enable_deletion: Optional[bool] = Field(..., alias="enableDeletion")
129
+ deletion_approvals: Optional[list[DeletionApprovalV1]] = Field(..., alias="deletionApprovals")
130
+ disable: Optional[DisableClusterAutomationsV1] = Field(..., alias="disable")
131
+ delete_keys: Optional[list[str]] = Field(..., alias="deleteKeys")
132
+ premium_support: bool = Field(..., alias="premiumSupport")
133
+ ecrs: Optional[list[AWSECRV1]] = Field(..., alias="ecrs")
134
+ partition: Optional[str] = Field(..., alias="partition")
135
+ cleanup: Optional[list[Union[AWSAccountCleanupOptionCloudWatchV1, AWSAccountCleanupOptionV1]]] = Field(..., alias="cleanup")
136
+
137
+
138
+ class AWSAccountsCloudwatchLogRetentionCleanupQueryData(ConfiguredBaseModel):
139
+ accounts: Optional[list[AWSAccountV1]] = Field(..., alias="accounts")
140
+
141
+
142
+ def query(query_func: Callable, **kwargs: Any) -> AWSAccountsCloudwatchLogRetentionCleanupQueryData:
143
+ """
144
+ This is a convenience function which queries and parses the data into
145
+ concrete types. It should be compatible with most GQL clients.
146
+ You do not have to use it to consume the generated data classes.
147
+ Alternatively, you can also mime and alternate the behavior
148
+ of this function in the caller.
149
+
150
+ Parameters:
151
+ query_func (Callable): Function which queries your GQL Server
152
+ kwargs: optional arguments that will be passed to the query function
153
+
154
+ Returns:
155
+ AWSAccountsCloudwatchLogRetentionCleanupQueryData: queried data parsed into generated classes
156
+ """
157
+ raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
158
+ return AWSAccountsCloudwatchLogRetentionCleanupQueryData(**raw_data)