qontract-reconcile 0.10.2.dev160__py3-none-any.whl → 0.10.2.dev162__py3-none-any.whl

{qontract_reconcile-0.10.2.dev160.dist-info → qontract_reconcile-0.10.2.dev162.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev160
+Version: 0.10.2.dev162
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile
@@ -20,7 +20,7 @@ Requires-Dist: croniter<1.1.0,>=1.0.15
 Requires-Dist: dateparser~=1.1.7
 Requires-Dist: deepdiff==6.7.1
 Requires-Dist: dnspython~=2.1
-Requires-Dist: dt==1.1.61
+Requires-Dist: dt==1.1.73
 Requires-Dist: filetype~=1.2.0
 Requires-Dist: gql==3.1.0
 Requires-Dist: hvac<0.8.0,>=0.7.0

{qontract_reconcile-0.10.2.dev160.dist-info → qontract_reconcile-0.10.2.dev162.dist-info}/RECORD

@@ -10,7 +10,7 @@ reconcile/aws_iam_password_reset.py,sha256=O0JX2N5kNRKs3u2xzu4NNrI6p0ag5JWy3MTsv
 reconcile/aws_support_cases_sos.py,sha256=PDhilxQ4TBxVnxUPIUdTbKEaNUI0wzPiEsB91oHT2fY,3384
 reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=O1wFp52EyF538c6txaWBs8eMtUIy19gyHZ6VzJ6QXS8,3512
 reconcile/checkpoint.py,sha256=_JhMxrye5BgkRMxWYuf7Upli6XayPINKSsuo3ynHTRc,5010
-reconcile/cli.py,sha256=hwqPcZVmazrhzq1esPBk5jNHSzpfr7o9EmuuMqiPxfg,108430
+reconcile/cli.py,sha256=xyVnxNyq3IPISWwFlB9j4HAFjowXYv3EdsEGIMFhTy0,108438
 reconcile/closedbox_endpoint_monitoring_base.py,sha256=al7m8EgnnYx90rY1REryW3byN_ItfJfAzEeLtjbCfi0,4921
 reconcile/cluster_deployment_mapper.py,sha256=5gumAaRCcFXsabUJ1dnuUy9WrP_FEEM5JnOnE8ch9sE,2326
 reconcile/dashdotdb_base.py,sha256=83ZWIf5JJk3P_D69y2TmXRcQr6ELJGlv10OM0h7fJVs,4767
@@ -22,7 +22,7 @@ reconcile/database_access_manager.py,sha256=Z3aAmw2LsmMIIor-bOGzziVZdVNC82Gmw8oH
 reconcile/deadmanssnitch.py,sha256=n-5W-djUgwzpmdDM4eQIZpkkDmHY0vndt-42LJXI4Y8,7491
 reconcile/email_sender.py,sha256=38Wvl6WHqCwlqLx4oxVJOIeDmoJsyitD3g1F4jTkAj8,4246
 reconcile/gabi_authorized_users.py,sha256=Jwvo97nzUX3NIl2VHKuZlT0-I40qk2VnACbafe91T2o,4854
-reconcile/gcr_mirror.py,sha256=cdTd0CZU0qUsXJqe5k4dgpMQlyk__nyeGH0f_Cky3C0,8957
+reconcile/gcp_image_mirror.py,sha256=M5pimd0j13BBWCa8vX0fftWO0pHBfQCATIIpOGggDSA,10332
 reconcile/github_org.py,sha256=Wc5cZamatuWsW2ZJT2ib5ps8l3iY3RXHwNUxVJerqz0,14173
 reconcile/github_owners.py,sha256=viE1KJ-zaTxuZ5yItg2C263J0brn-Q-3hR_DkYDMbhY,3122
 reconcile/github_repo_invites.py,sha256=U9UCzNVwrZ7MqODtFah8ogH0NNY-XjBin7G9gqHtCUY,2690
@@ -227,7 +227,7 @@ reconcile/glitchtip_project_alerts/integration.py,sha256=BgMx-NyV9mTuv7Sotb2OioC
 reconcile/glitchtip_project_dsn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/glitchtip_project_dsn/integration.py,sha256=2iugub-kHYkHNK33n0v9_TeWonuxCPah_VkoTPvaajE,8077
 reconcile/gql_definitions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/gql_definitions/introspection.json,sha256=myUuHC_BLY3xZ0nDjnaQsvYFNM5eTiE3bWgNgM3e5iI,2290863
+reconcile/gql_definitions/introspection.json,sha256=wNGZv8V6ivviCcwPLw34nzTR5fgwKRyE68Z93HbmBBs,2296576
 reconcile/gql_definitions/acs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/acs/acs_instances.py,sha256=L91WW9LbhJbBSrECqShQpFtjoBOsmNIYLRpMbx1io5o,2181
 reconcile/gql_definitions/acs/acs_policies.py,sha256=bN5i4mks10Z23KJSj7jqp966Osq2dps4d-sPH9gjxEA,7008
@@ -335,6 +335,7 @@ reconcile/gql_definitions/fragments/aws_infra_management_account.py,sha256=uAmAL
 reconcile/gql_definitions/fragments/aws_vpc.py,sha256=T2egTwi2Rb0IRBBmsyag8xKpu_m6GbIAy80fhZNZwk8,1434
 reconcile/gql_definitions/fragments/aws_vpc_request.py,sha256=o0qUsPrFXs8GAbtgMXQmIJxc1mw5skSIzCcidE857g8,2460
 reconcile/gql_definitions/fragments/aws_vpc_request_subnet.py,sha256=qaTFT8cGzEslw51nUeb45Nfnv6kFxUm4CWrRR3xfBvA,760
+reconcile/gql_definitions/fragments/container_image_mirror.py,sha256=qyfQlnKUCzFEPgUJ9VGmDYFmiGHR7VZ_YJNd4KeoolM,968
 reconcile/gql_definitions/fragments/deplopy_resources.py,sha256=0u3xYqL5NpMf149BJLfPhHqAOWu06aLULdNk_2Mulxg,1089
 reconcile/gql_definitions/fragments/disable.py,sha256=Ojw98OSxcovrtmw_aAyhaVHhIa1MSUbBfKX4i2IpI74,715
 reconcile/gql_definitions/fragments/email_service.py,sha256=0wKpICsg4pcMfr2lszvnqbuPX7wVYoJ5cYFU2uQkHbY,803
@@ -355,6 +356,9 @@ reconcile/gql_definitions/fragments/terraform_state.py,sha256=S5QuTR9YlvUObiU7he
 reconcile/gql_definitions/fragments/upgrade_policy.py,sha256=cVza8zfra1E3yBsHiS-hKbys17fvv572GFnKshJjluE,1246
 reconcile/gql_definitions/fragments/user.py,sha256=TZyFEs1fBg5PkvWdyCxFDZ_3aRhcQzusfhObXFiOU_0,1025
 reconcile/gql_definitions/fragments/vault_secret.py,sha256=8xoQJNx1jKw_1yradq1iLEYWzuOHra1bEHHU7WHKxqo,833
+reconcile/gql_definitions/gcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/gql_definitions/gcp/gcp_docker_repos.py,sha256=HvNaJxQYNPBTDmk26cOUY5_C5oBfau4bdfuI-L1Vcps,3338
+reconcile/gql_definitions/gcp/gcp_projects.py,sha256=7LslYFSN2r8vYhYdi4s-zOkIrqpqyt4ayxbPNMie9M4,2108
 reconcile/gql_definitions/gitlab_members/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/gitlab_members/gitlab_instances.py,sha256=oYPvfiOsPTGHXQeSfxXvBuvJFrwp0VtE2F0lVKFQMoU,2206
 reconcile/gql_definitions/gitlab_members/permissions.py,sha256=Qzj3Fpv7xj8v9eygeP312nHRNg8er8XMRBveynPIyQM,3302
@@ -687,7 +691,7 @@ reconcile/utils/clusterhealth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5N
 reconcile/utils/clusterhealth/providerbase.py,sha256=DXomGYogckBLqWtXn0PXU0hWYxB6K0F7ernldrkHhVY,1140
 reconcile/utils/clusterhealth/telemeter.py,sha256=PllSLsJXvGNatmTF4mxCNPVbDrpr_MPk0m5pWj-LT6g,1534
 reconcile/utils/dynatrace/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/utils/dynatrace/client.py,sha256=RUk6KH-3CJyfJ1jolrdGQR4Hhz-tIWWJo9dsZ1IgJVw,3736
+reconcile/utils/dynatrace/client.py,sha256=H8EjqmZlB1a2ionAjV8_R1ozs9lWbmPCYKLe0J8kZAs,2838
 reconcile/utils/glitchtip/__init__.py,sha256=FT6iBhGqoe7KExFdbgL8AYUb64iW_4snF5__Dcl7yt0,258
 reconcile/utils/glitchtip/client.py,sha256=F_x18QMHfeNoS3vS_VvjVm1IKnwGOsXvopjR9iwj4aY,8948
 reconcile/utils/glitchtip/models.py,sha256=FyNt9EA9IS6tlsvqz-j7SkL9MoT_zIUTun0EiC9No6U,6684
@@ -800,7 +804,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
-qontract_reconcile-0.10.2.dev160.dist-info/METADATA,sha256=q307B4-jiPGwEIJonpzw-vEkV7sOmCmPOxubWtR60TI,24627
-qontract_reconcile-0.10.2.dev160.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-qontract_reconcile-0.10.2.dev160.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
-qontract_reconcile-0.10.2.dev160.dist-info/RECORD,,
+qontract_reconcile-0.10.2.dev162.dist-info/METADATA,sha256=HuDQ0XJUu4_ZT4LGmWTJVTTnfcqy21AIo-LCxVesVYo,24627
+qontract_reconcile-0.10.2.dev162.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+qontract_reconcile-0.10.2.dev162.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
+qontract_reconcile-0.10.2.dev162.dist-info/RECORD,,

reconcile/cli.py

@@ -1849,15 +1849,13 @@ def quay_membership(ctx):
     run_integration(reconcile.quay_membership, ctx.obj)
 
 
-@integration.command(
-    short_help="Mirrors external images into Google Container Registry."
-)
+@integration.command(short_help="Mirrors external images into GCP Artifact Registry.")
 @click.pass_context
 @binary(["skopeo"])
-def gcr_mirror(ctx):
-    import reconcile.gcr_mirror
+def gcp_image_mirror(ctx):
+    import reconcile.gcp_image_mirror
 
-    run_integration(reconcile.gcr_mirror, ctx.obj)
+    run_integration(reconcile.gcp_image_mirror, ctx.obj)
 
 
 @integration.command(short_help="Mirrors external images into Quay.")

reconcile/gcp_image_mirror.py

@@ -0,0 +1,276 @@
+import base64
+import logging
+import os
+import re
+import tempfile
+import time
+from typing import Any, Self
+
+import requests
+from pydantic import BaseModel
+from sretoolbox.container import (
+    Image,
+    Skopeo,
+)
+from sretoolbox.container.image import ImageComparisonError
+from sretoolbox.container.skopeo import SkopeoCmdError
+
+import reconcile.gql_definitions.gcp.gcp_docker_repos as gql_gcp_repos
+import reconcile.gql_definitions.gcp.gcp_projects as gql_gcp_projects
+from reconcile import queries
+from reconcile.gql_definitions.fragments.container_image_mirror import (
+    ContainerImageMirror,
+)
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+from reconcile.utils import gql
+from reconcile.utils.secret_reader import SecretReader
+
+QONTRACT_INTEGRATION = "gcp-image-mirror"
+REQUEST_TIMEOUT = 60
+GCR_SECRET_PREFIX = "gcr_"
+AR_SECRET_PREFIX = "ar_"
+
+
+class ImageSyncItem(BaseModel):
+    mirror: ContainerImageMirror
+    destination_url: str
+    org_name: str
+
+
+class SyncTask(BaseModel):
+    mirror_creds: str | None = None
+    source_url: str
+    dest_url: str
+    org_name: str
+
+
+class QuayMirror:
+    def __init__(self, dry_run: bool = False) -> None:
+        self.dry_run = dry_run
+        self.gqlapi = gql.get_api()
+        settings = queries.get_app_interface_settings()
+        self.secret_reader = SecretReader(settings=settings)
+        self.skopeo_cli = Skopeo(dry_run)
+        self.push_creds = self._get_push_creds()
+        self.session = requests.Session()
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        self.session.close()
+
+    def run(self) -> None:
+        gql_result = gql_gcp_repos.query(query_func=self.gqlapi.query)
+        processed_repos = self.process_repos_to_sync(gql_result)
+        sync_tasks = self.process_sync_tasks(processed_repos)
+
+        for task in sync_tasks:
+            try:
+                dest_creds = self.push_creds[f"{GCR_SECRET_PREFIX}{task.org_name}"]
+                if "pkg.dev" in task.dest_url:
+                    dest_creds = self.push_creds[f"{AR_SECRET_PREFIX}{task.org_name}"]
+
+                self.skopeo_cli.copy(
+                    src_image=task.source_url,
+                    src_creds=task.mirror_creds,
+                    dst_image=task.dest_url,
+                    dest_creds=dest_creds,
+                )
+            except SkopeoCmdError as details:
+                logging.error("[%s]", details)
+
+    # processes the GQL repos to come up with a list of items that need to be synced
+    def process_repos_to_sync(
+        self, repos: gql_gcp_repos.GcpDockerReposQueryData
+    ) -> list[ImageSyncItem]:
+        summary = list[ImageSyncItem]()
+        if repos.apps:
+            for app in repos.apps:
+                if app.gcr_repos:
+                    for gcr_project in app.gcr_repos:
+                        for gcr_repo in gcr_project.items:
+                            if gcr_repo.mirror:
+                                project_name = gcr_project.project.name
+                                summary.append(
+                                    ImageSyncItem(
+                                        mirror=gcr_repo.mirror,
+                                        destination_url=f"gcr.io/{project_name}/{gcr_repo.name}",
+                                        org_name=project_name,
+                                    )
+                                )
+                if app.artifact_registry_mirrors:
+                    for ar_project in app.artifact_registry_mirrors:
+                        for ar_repo in ar_project.items:
+                            summary.append(
+                                ImageSyncItem(
+                                    mirror=ar_repo.mirror,
+                                    destination_url=ar_repo.image_url,
+                                    org_name=ar_project.project.name,
+                                )
+                            )
+
+        return summary
+
+    @staticmethod
+    def sync_tag(
+        tags: list[str] | None, tags_exclude: list[str] | None, candidate: str
+    ) -> bool:
+        if tags is not None:
+            # When tags is defined, we don't look at tags_exclude
+            return any(re.match(tag, candidate) for tag in tags)
+
+        if tags_exclude is not None:
+            return any(re.match(tag, candidate) for tag in tags_exclude)
+            for tag_exclude in tags_exclude:
+                if re.match(tag_exclude, candidate):
+                    return False
+            return True
+
+        # Both tags and tags_exclude are None, so
+        # tag must be synced
+        return True
+
+    # second layer of processing that matches up pull/push creds with each repo and determines what tags need to be synced
+    def process_sync_tasks(self, repos_to_sync: list[ImageSyncItem]) -> list[SyncTask]:
+        eight_hours = 28800  # 60 * 60 * 8
+        is_deep_sync = self._is_deep_sync(interval=eight_hours)
+
+        sync_tasks = list[SyncTask]()
+        for item in repos_to_sync:
+            image = Image(
+                f"{item.destination_url}",
+                session=self.session,
+                timeout=REQUEST_TIMEOUT,
+            )
+
+            mirror_url = item.mirror.url
+
+            username = None
+            password = None
+            mirror_creds = None
+            pull_credentials = item.mirror.pull_credentials
+            if pull_credentials:
+                raw_data = self.secret_reader.read_all(pull_credentials.dict())
+                username = raw_data["user"]
+                password = raw_data["token"]
+                mirror_creds = f"{username}:{password}"
+
+            image_mirror = Image(
+                mirror_url,
+                username=username,
+                password=password,
+                session=self.session,
+                timeout=REQUEST_TIMEOUT,
+            )
+
+            for tag in image_mirror:
+                if not self.sync_tag(
+                    tags=item.mirror.tags,
+                    tags_exclude=item.mirror.tags_exclude,
+                    candidate=tag,
+                ):
+                    continue
+
+                # the Image class allows you to fetch Image information at a specific tag with a get operator
+                upstream = image_mirror[tag]
+                downstream = image[tag]
+                if tag not in image:
+                    logging.debug(
+                        f"Image {image.image}: {downstream} and mirror {upstream} are out of sync"
+                    )
+                    sync_tasks.append(
+                        SyncTask(
+                            source_url=str(upstream),
+                            mirror_creds=mirror_creds,
+                            dest_url=str(downstream),
+                            org_name=item.org_name,
+                        )
+                    )
+                    continue
+
+                # Deep (slow) check only in non dry-run mode
+                if self.dry_run:
+                    logging.debug(
+                        f"Image {image.image}: {downstream} and mirror {upstream} are in sync"
+                    )
+                    continue
+
+                # Deep (slow) check only from time to time
+                if not is_deep_sync:
+                    logging.debug(
+                        f"Image {image.image}: {downstream} and mirror {upstream} are in sync"
+                    )
+                    continue
+
+                try:
+                    if downstream == upstream:
+                        logging.debug(
+                            f"Image {image.image}: {downstream} and mirror {upstream} are in sync",
+                        )
+                        continue
+                except ImageComparisonError as details:
+                    logging.error("[%s]", details)
+                    continue
+
+                logging.debug(
+                    f"Image {image.image}: {downstream} and mirror {upstream} are out of sync"
+                )
+                sync_tasks.append(
+                    SyncTask(
+                        source_url=str(upstream),
+                        mirror_creds=mirror_creds,
+                        dest_url=str(downstream),
+                        org_name=item.org_name,
+                    )
+                )
+
+        return sync_tasks
+
+    def _is_deep_sync(self, interval: int) -> bool:
+        control_file_name = "qontract-reconcile-gcp-image-mirror.timestamp"
+        control_file_path = os.path.join(tempfile.gettempdir(), control_file_name)
+        try:
+            with open(control_file_path, encoding="locale") as file_obj:
+                last_deep_sync = float(file_obj.read())
+        except FileNotFoundError:
+            self._record_timestamp(control_file_path)
+            return True
+
+        next_deep_sync = last_deep_sync + interval
+        if time.time() >= next_deep_sync:
+            self._record_timestamp(control_file_path)
+            return True
+
+        return False
+
+    def _decode_push_secret(self, secret: VaultSecret) -> str:
+        raw_data = self.secret_reader.read_all(secret.dict())
+        token = base64.b64decode(raw_data["token"]).decode()
+        return f"{raw_data['user']}:{token}"
+
+    @staticmethod
+    def _record_timestamp(path: str) -> None:
+        with open(path, "w", encoding="locale") as file_object:
+            file_object.write(str(time.time()))
+
+    def _get_push_creds(self) -> dict[str, str]:
+        result = gql_gcp_projects.query(query_func=self.gqlapi.query)
+
+        creds = dict[str, str]()
+        if result.gcp_projects:
+            for project_data in result.gcp_projects:
+                # support old pull secret for backwards compatibility (although they are both using artifact registry on the backend)
+                if project_data.gcr_push_credentials:
+                    creds[f"{GCR_SECRET_PREFIX}{project_data.name}"] = (
+                        self._decode_push_secret(project_data.gcr_push_credentials)
+                    )
+                creds[f"{AR_SECRET_PREFIX}{project_data.name}"] = (
+                    self._decode_push_secret(project_data.artifact_push_credentials)
+                )
+        return creds
+
+
+def run(dry_run: bool) -> None:
+    with QuayMirror(dry_run) as gcp_image_mirror:
+        gcp_image_mirror.run()

reconcile/gql_definitions/fragments/container_image_mirror.py

@@ -0,0 +1,33 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class ContainerImageMirror(ConfiguredBaseModel):
+    url: str = Field(..., alias="url")
+    pull_credentials: Optional[VaultSecret] = Field(..., alias="pullCredentials")
+    tags: Optional[list[str]] = Field(..., alias="tags")
+    tags_exclude: Optional[list[str]] = Field(..., alias="tagsExclude")

reconcile/gql_definitions/gcp/gcp_docker_repos.py

@@ -0,0 +1,128 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.container_image_mirror import ContainerImageMirror
+
+
+DEFINITION = """
+fragment ContainerImageMirror on ContainerImageMirror_v1 {
+  url
+  pullCredentials {
+    ...VaultSecret
+  }
+  tags
+  tagsExclude
+}
+
+fragment VaultSecret on VaultSecret_v1 {
+    path
+    field
+    version
+    format
+}
+
+query GcpDockerRepos {
+  apps: apps_v1 {
+    gcrRepos {
+      project {
+        name
+      }
+      items {
+        name
+        mirror {
+          ...ContainerImageMirror
+        }
+      }
+    }
+    artifactRegistryMirrors {
+      project {
+        name
+      }
+      items {
+        imageURL
+        mirror {
+          ...ContainerImageMirror
+        }
+      }
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class GcpProjectV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+
+
+class AppGcrReposItemsV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    mirror: Optional[ContainerImageMirror] = Field(..., alias="mirror")
+
+
+class AppGcrReposV1(ConfiguredBaseModel):
+    project: GcpProjectV1 = Field(..., alias="project")
+    items: list[AppGcrReposItemsV1] = Field(..., alias="items")
+
+
+class AppArtifactRegistryMirrorsV1_GcpProjectV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+
+
+class AppArtifactRegistryMirrorsItemsV1(ConfiguredBaseModel):
+    image_url: str = Field(..., alias="imageURL")
+    mirror: ContainerImageMirror = Field(..., alias="mirror")
+
+
+class AppArtifactRegistryMirrorsV1(ConfiguredBaseModel):
+    project: AppArtifactRegistryMirrorsV1_GcpProjectV1 = Field(..., alias="project")
+    items: list[AppArtifactRegistryMirrorsItemsV1] = Field(..., alias="items")
+
+
+class AppV1(ConfiguredBaseModel):
+    gcr_repos: Optional[list[AppGcrReposV1]] = Field(..., alias="gcrRepos")
+    artifact_registry_mirrors: Optional[list[AppArtifactRegistryMirrorsV1]] = Field(..., alias="artifactRegistryMirrors")
+
+
+class GcpDockerReposQueryData(ConfiguredBaseModel):
+    apps: Optional[list[AppV1]] = Field(..., alias="apps")
+
+
+def query(query_func: Callable, **kwargs: Any) -> GcpDockerReposQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        GcpDockerReposQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return GcpDockerReposQueryData(**raw_data)

reconcile/gql_definitions/gcp/gcp_projects.py

@@ -0,0 +1,77 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+
+
+DEFINITION = """
+fragment VaultSecret on VaultSecret_v1 {
+    path
+    field
+    version
+    format
+}
+
+query GcpProjects {
+  gcp_projects: gcp_projects_v1 {
+    name
+    gcrPushCredentials {
+      ...VaultSecret
+    }
+    artifactPushCredentials {
+      ...VaultSecret
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class GcpProjectV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    gcr_push_credentials: Optional[VaultSecret] = Field(..., alias="gcrPushCredentials")
+    artifact_push_credentials: VaultSecret = Field(..., alias="artifactPushCredentials")
+
+
+class GcpProjectsQueryData(ConfiguredBaseModel):
+    gcp_projects: Optional[list[GcpProjectV1]] = Field(..., alias="gcp_projects")
+
+
+def query(query_func: Callable, **kwargs: Any) -> GcpProjectsQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        GcpProjectsQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return GcpProjectsQueryData(**raw_data)

reconcile/gql_definitions/introspection.json

@@ -15959,6 +15959,26 @@
                             "isDeprecated": false,
                             "deprecationReason": null
                         },
+                        {
+                            "name": "artifactRegistryMirrors",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "LIST",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "NON_NULL",
+                                    "name": null,
+                                    "ofType": {
+                                        "kind": "OBJECT",
+                                        "name": "AppArtifactRegistryMirrors_v1",
+                                        "ofType": null
+                                    }
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
                         {
                             "name": "quayRepos",
                             "description": null,
@@ -16816,7 +16836,7 @@
                             "deprecationReason": null
                         },
                         {
-                            "name": "pushCredentials",
+                            "name": "gcrPushCredentials",
                             "description": null,
                             "args": [],
                             "type": {
@@ -16826,6 +16846,22 @@
                             },
                             "isDeprecated": false,
                             "deprecationReason": null
+                        },
+                        {
+                            "name": "artifactPushCredentials",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "NON_NULL",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "OBJECT",
+                                    "name": "VaultSecret_v1",
+                                    "ofType": null
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
                         }
                     ],
                     "inputFields": null,
@@ -17027,6 +17063,100 @@
                     "enumValues": null,
                     "possibleTypes": null
                 },
+                {
+                    "kind": "OBJECT",
+                    "name": "AppArtifactRegistryMirrors_v1",
+                    "description": null,
+                    "fields": [
+                        {
+                            "name": "project",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "NON_NULL",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "OBJECT",
+                                    "name": "GcpProject_v1",
+                                    "ofType": null
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "items",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "NON_NULL",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "LIST",
+                                    "name": null,
+                                    "ofType": {
+                                        "kind": "NON_NULL",
+                                        "name": null,
+                                        "ofType": {
+                                            "kind": "OBJECT",
+                                            "name": "AppArtifactRegistryMirrorsItems_v1",
+                                            "ofType": null
+                                        }
+                                    }
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        }
+                    ],
+                    "inputFields": null,
+                    "interfaces": [],
+                    "enumValues": null,
+                    "possibleTypes": null
+                },
+                {
+                    "kind": "OBJECT",
+                    "name": "AppArtifactRegistryMirrorsItems_v1",
+                    "description": null,
+                    "fields": [
+                        {
+                            "name": "imageURL",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "NON_NULL",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "SCALAR",
+                                    "name": "String",
+                                    "ofType": null
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "mirror",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "NON_NULL",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "OBJECT",
+                                    "name": "ContainerImageMirror_v1",
+                                    "ofType": null
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        }
+                    ],
+                    "inputFields": null,
+                    "interfaces": [],
+                    "enumValues": null,
+                    "possibleTypes": null
+                },
                 {
                     "kind": "OBJECT",
                     "name": "AppQuayRepos_v1",

reconcile/utils/dynatrace/client.py

@@ -1,8 +1,6 @@
 from __future__ import annotations
 
 from collections.abc import Iterable
-from datetime import datetime
-from unittest.mock import patch
 
 from dynatrace import Dynatrace
 from dynatrace.environment_v2.tokens_api import ApiTokenUpdate
@@ -35,32 +33,11 @@ class DynatraceAPIToken(BaseModel):
     scopes: list[str]
 
 
-# TODO: Remove once APPSRE-11428 is resolved #######
-ISO_8601 = "%Y-%m-%dT%H:%M:%S.%fZ"
-FIXED_ISO_8601 = "%Y-%m-%dT%H:%M:%SZ"
-
-
-def custom_iso8601_to_datetime(timestamp: str | None) -> datetime | None:
-    if isinstance(timestamp, str):
-        try:
-            return datetime.strptime(timestamp, ISO_8601)
-        except ValueError:
-            return datetime.strptime(timestamp, FIXED_ISO_8601)
-    return timestamp
-
-
-################################################
-
-
 class DynatraceClient:
     def __init__(self, environment_url: str, api: Dynatrace) -> None:
         self._environment_url = environment_url
         self._api = api
 
-    @patch(
-        "dynatrace.environment_v2.tokens_api.iso8601_to_datetime",
-        custom_iso8601_to_datetime,
-    )
     def create_api_token(
         self, name: str, scopes: Iterable[str]
     ) -> DynatraceAPITokenCreated:
@@ -72,10 +49,6 @@ class DynatraceClient:
             ) from e
         return DynatraceAPITokenCreated(token=token.token, id=token.id)
 
-    @patch(
-        "dynatrace.environment_v2.tokens_api.iso8601_to_datetime",
-        custom_iso8601_to_datetime,
-    )
     def get_token_ids_map_for_name_prefix(self, prefix: str) -> dict[str, str]:
         try:
             dt_tokens = self._api.tokens.list()
@@ -87,10 +60,6 @@ class DynatraceClient:
             token.id: token.name for token in dt_tokens if token.name.startswith(prefix)
         }
 
-    @patch(
-        "dynatrace.environment_v2.tokens_api.iso8601_to_datetime",
-        custom_iso8601_to_datetime,
-    )
     def get_token_by_id(self, token_id: str) -> DynatraceAPIToken:
         try:
             token = self._api.tokens.get(token_id=token_id)

reconcile/gcr_mirror.py

@@ -1,278 +0,0 @@
-import base64
-import logging
-import os
-import re
-import tempfile
-import time
-from collections import defaultdict
-from typing import Any, Self
-
-import requests
-from sretoolbox.container import (
-    Image,
-    Skopeo,
-)
-from sretoolbox.container.image import ImageComparisonError
-from sretoolbox.container.skopeo import SkopeoCmdError
-
-from reconcile import queries
-from reconcile.utils import gql
-from reconcile.utils.secret_reader import SecretReader
-
-_LOG = logging.getLogger(__name__)
-
-QONTRACT_INTEGRATION = "gcr-mirror"
-REQUEST_TIMEOUT = 60
-
-
-class QuayMirror:
-    GCR_PROJECT_CATALOG_QUERY = """
-    {
-      projects: gcp_projects_v1 {
-        name
-        pushCredentials {
-          path
-          field
-          version
-          format
-        }
-      }
-    }
-    """
-
-    GCR_REPOS_QUERY = """
-    {
-      apps: apps_v1 {
-        gcrRepos {
-          project {
-            name
-          }
-          items {
-            name
-            mirror {
-              url
-              pullCredentials {
-                path
-                field
-                version
-                format
-              }
-              tags
-              tagsExclude
-            }
-          }
-        }
-      }
-    }
-    """
-
-    def __init__(self, dry_run: bool = False) -> None:
-        self.dry_run = dry_run
-        self.gqlapi = gql.get_api()
-        settings = queries.get_app_interface_settings()
-        self.secret_reader = SecretReader(settings=settings)
-        self.skopeo_cli = Skopeo(dry_run)
-        self.push_creds = self._get_push_creds()
-        self.session = requests.Session()
-
-    def __enter__(self) -> Self:
-        return self
-
-    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
-        self.session.close()
-
-    def run(self) -> None:
-        sync_tasks = self.process_sync_tasks()
-        for org, data in sync_tasks.items():
-            for item in data:
-                try:
-                    self.skopeo_cli.copy(
-                        src_image=item["mirror_url"],
-                        src_creds=item["mirror_creds"],
-                        dst_image=item["image_url"],
-                        dest_creds=self.push_creds[org],
-                    )
-                except SkopeoCmdError as details:
-                    _LOG.error("[%s]", details)
-
-    def process_repos_query(self) -> dict[str, list[dict[str, Any]]]:
-        result = self.gqlapi.query(self.GCR_REPOS_QUERY)
-
-        summary = defaultdict(list)
-
-        for app in result["apps"]:
-            gcr_repos = app.get("gcrRepos")
-
-            if gcr_repos is None:
-                continue
-
-            for gcr_repo in gcr_repos:
-                project = gcr_repo["project"]["name"]
-                server_url = gcr_repo["project"].get("serverUrl") or "gcr.io"
-                for item in gcr_repo["items"]:
-                    if item["mirror"] is None:
-                        continue
-
-                    summary[project].append({
-                        "name": item["name"],
-                        "mirror": item["mirror"],
-                        "server_url": server_url,
-                    })
-
-        return summary
-
-    @staticmethod
-    def sync_tag(
-        tags: list[str] | None, tags_exclude: list[str] | None, candidate: str
-    ) -> bool:
-        if tags is not None:
-            # When tags is defined, we don't look at tags_exclude
-            return any(re.match(tag, candidate) for tag in tags)
-
-        if tags_exclude is not None:
-            for tag_exclude in tags_exclude:
-                if re.match(tag_exclude, candidate):
-                    return False
-            return True
-
-        # Both tags and tags_exclude are None, so
-        # tag must be synced
-        return True
-
-    def process_sync_tasks(self) -> dict[str, list[dict[str, Any]]]:
-        eight_hours = 28800  # 60 * 60 * 8
-        is_deep_sync = self._is_deep_sync(interval=eight_hours)
-
-        summary = self.process_repos_query()
-
-        sync_tasks = defaultdict(list)
-        for org, data in summary.items():
-            for item in data:
-                image = Image(
-                    f"{item['server_url']}/{org}/{item['name']}",
-                    session=self.session,
-                    timeout=REQUEST_TIMEOUT,
-                )
-
-                mirror_url = item["mirror"]["url"]
-
-                username = None
-                password = None
-                mirror_creds = None
-                if item["mirror"]["pullCredentials"] is not None:
-                    pull_credentials = item["mirror"]["pullCredentials"]
-                    raw_data = self.secret_reader.read_all(pull_credentials)
-                    username = raw_data["user"]
-                    password = raw_data["token"]
-                    mirror_creds = f"{username}:{password}"
-
-                image_mirror = Image(
-                    mirror_url,
-                    username=username,
-                    password=password,
-                    session=self.session,
-                    timeout=REQUEST_TIMEOUT,
-                )
-
-                tags = item["mirror"].get("tags")
-                tags_exclude = item["mirror"].get("tagsExclude")
-
-                for tag in image_mirror:
-                    if not self.sync_tag(
-                        tags=tags, tags_exclude=tags_exclude, candidate=tag
-                    ):
-                        continue
-
-                    upstream = image_mirror[tag]
-                    downstream = image[tag]
-                    if tag not in image:
-                        _LOG.debug(
-                            "Image %s and mirror %s are out off sync",
-                            downstream,
-                            upstream,
-                        )
-                        sync_tasks[org].append({
-                            "mirror_url": str(upstream),
-                            "mirror_creds": mirror_creds,
-                            "image_url": str(downstream),
-                        })
-                        continue
-
-                    # Deep (slow) check only in non dry-run mode
-                    if self.dry_run:
-                        _LOG.debug(
-                            "Image %s and mirror %s are in sync", downstream, upstream
-                        )
-                        continue
-
-                    # Deep (slow) check only from time to time
-                    if not is_deep_sync:
-                        _LOG.debug(
-                            "Image %s and mirror %s are in sync", downstream, upstream
-                        )
-                        continue
-
-                    try:
-                        if downstream == upstream:
-                            _LOG.debug(
-                                "Image %s and mirror %s are in sync",
-                                downstream,
-                                upstream,
-                            )
-                            continue
-                    except ImageComparisonError as details:
-                        _LOG.error("[%s]", details)
-                        continue
-
-                    _LOG.debug(
-                        "Image %s and mirror %s are out of sync", downstream, upstream
-                    )
-                    sync_tasks[org].append({
-                        "mirror_url": str(upstream),
-                        "mirror_creds": mirror_creds,
-                        "image_url": str(downstream),
-                    })
-
-        return sync_tasks
-
-    def _is_deep_sync(self, interval: int) -> bool:
-        control_file_name = "qontract-reconcile-gcr-mirror.timestamp"
-        control_file_path = os.path.join(tempfile.gettempdir(), control_file_name)
-        try:
-            with open(control_file_path, encoding="locale") as file_obj:
-                last_deep_sync = float(file_obj.read())
-        except FileNotFoundError:
-            self._record_timestamp(control_file_path)
-            return True
-
-        next_deep_sync = last_deep_sync + interval
-        if time.time() >= next_deep_sync:
-            self._record_timestamp(control_file_path)
-            return True
-
-        return False
-
-    @staticmethod
-    def _record_timestamp(path: str) -> None:
-        with open(path, "w", encoding="locale") as file_object:
-            file_object.write(str(time.time()))
-
-    def _get_push_creds(self) -> dict[str, str]:
-        result = self.gqlapi.query(self.GCR_PROJECT_CATALOG_QUERY)
-
-        creds = {}
-        for project_data in result["projects"]:
-            push_secret = project_data["pushCredentials"]
-            if push_secret is None:
-                continue
-
-            raw_data = self.secret_reader.read_all(push_secret)
-            project = project_data["name"]
-            token = base64.b64decode(raw_data["token"]).decode()
-            creds[project] = f"{raw_data['user']}:{token}"
-        return creds
-
-
-def run(dry_run: bool) -> None:
-    with QuayMirror(dry_run) as gcr_mirror:
-        gcr_mirror.run()