qontract-reconcile 0.10.2.dev160__py3-none-any.whl → 0.10.2.dev173__py3-none-any.whl

reconcile/gcp_image_mirror.py

@@ -0,0 +1,252 @@
+import base64
+import logging
+import os
+import tempfile
+import time
+from typing import Any, Self
+
+import requests
+from pydantic import BaseModel
+from sretoolbox.container import (
+    Image,
+    Skopeo,
+)
+from sretoolbox.container.image import ImageComparisonError
+from sretoolbox.container.skopeo import SkopeoCmdError
+
+import reconcile.gql_definitions.gcp.gcp_docker_repos as gql_gcp_repos
+import reconcile.gql_definitions.gcp.gcp_projects as gql_gcp_projects
+from reconcile import queries
+from reconcile.gql_definitions.fragments.container_image_mirror import (
+    ContainerImageMirror,
+)
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+from reconcile.utils import gql
+from reconcile.utils.quay_mirror import record_timestamp, sync_tag
+from reconcile.utils.secret_reader import SecretReader
+
+QONTRACT_INTEGRATION = "gcp-image-mirror"
+REQUEST_TIMEOUT = 60
+GCR_SECRET_PREFIX = "gcr_"
+AR_SECRET_PREFIX = "ar_"
+
+
+class ImageSyncItem(BaseModel):
+    mirror: ContainerImageMirror
+    destination_url: str
+    org_name: str
+
+
+class SyncTask(BaseModel):
+    mirror_creds: str | None = None
+    source_url: str
+    dest_url: str
+    org_name: str
+
+
+class QuayMirror:
+    def __init__(self, dry_run: bool = False) -> None:
+        self.dry_run = dry_run
+        self.gqlapi = gql.get_api()
+        settings = queries.get_app_interface_settings()
+        self.secret_reader = SecretReader(settings=settings)
+        self.skopeo_cli = Skopeo(dry_run)
+        self.push_creds = self._get_push_creds()
+        self.session = requests.Session()
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        self.session.close()
+
+    def run(self) -> None:
+        gql_result = gql_gcp_repos.query(query_func=self.gqlapi.query)
+        processed_repos = self.process_repos_to_sync(gql_result)
+        sync_tasks = self.process_sync_tasks(processed_repos)
+
+        for task in sync_tasks:
+            try:
+                dest_creds = self.push_creds[f"{GCR_SECRET_PREFIX}{task.org_name}"]
+                if "pkg.dev" in task.dest_url:
+                    dest_creds = self.push_creds[f"{AR_SECRET_PREFIX}{task.org_name}"]
+
+                self.skopeo_cli.copy(
+                    src_image=task.source_url,
+                    src_creds=task.mirror_creds,
+                    dst_image=task.dest_url,
+                    dest_creds=dest_creds,
+                )
+            except SkopeoCmdError as details:
+                logging.error("[%s]", details)
+
+    # processes the GQL repos to come up with a list of items that need to be synced
+    def process_repos_to_sync(
+        self, repos: gql_gcp_repos.GcpDockerReposQueryData
+    ) -> list[ImageSyncItem]:
+        summary = list[ImageSyncItem]()
+        if repos.apps:
+            for app in repos.apps:
+                if app.gcr_repos:
+                    for gcr_project in app.gcr_repos:
+                        for gcr_repo in gcr_project.items:
+                            if gcr_repo.mirror:
+                                project_name = gcr_project.project.name
+                                summary.append(
+                                    ImageSyncItem(
+                                        mirror=gcr_repo.mirror,
+                                        destination_url=f"gcr.io/{project_name}/{gcr_repo.name}",
+                                        org_name=project_name,
+                                    )
+                                )
+                if app.artifact_registry_mirrors:
+                    for ar_project in app.artifact_registry_mirrors:
+                        for ar_repo in ar_project.items:
+                            summary.append(
+                                ImageSyncItem(
+                                    mirror=ar_repo.mirror,
+                                    destination_url=ar_repo.image_url,
+                                    org_name=ar_project.project.name,
+                                )
+                            )
+
+        return summary
+
+    # second layer of processing that matches up pull/push creds with each repo and determines what tags need to be synced
+    def process_sync_tasks(self, repos_to_sync: list[ImageSyncItem]) -> list[SyncTask]:
+        eight_hours = 28800  # 60 * 60 * 8
+        is_deep_sync = self._is_deep_sync(interval=eight_hours)
+
+        sync_tasks = list[SyncTask]()
+        for item in repos_to_sync:
+            image = Image(
+                f"{item.destination_url}",
+                session=self.session,
+                timeout=REQUEST_TIMEOUT,
+            )
+
+            mirror_url = item.mirror.url
+
+            username = None
+            password = None
+            mirror_creds = None
+            pull_credentials = item.mirror.pull_credentials
+            if pull_credentials:
+                raw_data = self.secret_reader.read_all(pull_credentials.dict())
+                username = raw_data["user"]
+                password = raw_data["token"]
+                mirror_creds = f"{username}:{password}"
+
+            image_mirror = Image(
+                mirror_url,
+                username=username,
+                password=password,
+                session=self.session,
+                timeout=REQUEST_TIMEOUT,
+            )
+
+            for tag in image_mirror:
+                if not sync_tag(
+                    tags=item.mirror.tags,
+                    tags_exclude=item.mirror.tags_exclude,
+                    candidate=tag,
+                ):
+                    continue
+
+                # the Image class allows you to fetch Image information at a specific tag with a get operator
+                upstream = image_mirror[tag]
+                downstream = image[tag]
+                if tag not in image:
+                    logging.debug(
+                        f"Image {image.image}: {downstream} and mirror {upstream} are out of sync"
+                    )
+                    sync_tasks.append(
+                        SyncTask(
+                            source_url=str(upstream),
+                            mirror_creds=mirror_creds,
+                            dest_url=str(downstream),
+                            org_name=item.org_name,
+                        )
+                    )
+                    continue
+
+                # Deep (slow) check only in non dry-run mode
+                if self.dry_run:
+                    logging.debug(
+                        f"Image {image.image}: {downstream} and mirror {upstream} are in sync"
+                    )
+                    continue
+
+                # Deep (slow) check only from time to time
+                if not is_deep_sync:
+                    logging.debug(
+                        f"Image {image.image}: {downstream} and mirror {upstream} are in sync"
+                    )
+                    continue
+
+                try:
+                    if downstream == upstream:
+                        logging.debug(
+                            f"Image {image.image}: {downstream} and mirror {upstream} are in sync",
+                        )
+                        continue
+                except ImageComparisonError as details:
+                    logging.error("[%s]", details)
+                    continue
+
+                logging.debug(
+                    f"Image {image.image}: {downstream} and mirror {upstream} are out of sync"
+                )
+                sync_tasks.append(
+                    SyncTask(
+                        source_url=str(upstream),
+                        mirror_creds=mirror_creds,
+                        dest_url=str(downstream),
+                        org_name=item.org_name,
+                    )
+                )
+
+        return sync_tasks
+
+    def _is_deep_sync(self, interval: int) -> bool:
+        control_file_name = "qontract-reconcile-gcp-image-mirror.timestamp"
+        control_file_path = os.path.join(tempfile.gettempdir(), control_file_name)
+        try:
+            with open(control_file_path, encoding="locale") as file_obj:
+                last_deep_sync = float(file_obj.read())
+        except FileNotFoundError:
+            record_timestamp(control_file_path)
+            return True
+
+        next_deep_sync = last_deep_sync + interval
+        if time.time() >= next_deep_sync:
+            record_timestamp(control_file_path)
+            return True
+
+        return False
+
+    def _decode_push_secret(self, secret: VaultSecret) -> str:
+        raw_data = self.secret_reader.read_all(secret.dict())
+        token = base64.b64decode(raw_data["token"]).decode()
+        return f"{raw_data['user']}:{token}"
+
+    def _get_push_creds(self) -> dict[str, str]:
+        result = gql_gcp_projects.query(query_func=self.gqlapi.query)
+
+        creds = dict[str, str]()
+        if result.gcp_projects:
+            for project_data in result.gcp_projects:
+                # support old pull secret for backwards compatibility (although they are both using artifact registry on the backend)
+                if project_data.gcr_push_credentials:
+                    creds[f"{GCR_SECRET_PREFIX}{project_data.name}"] = (
+                        self._decode_push_secret(project_data.gcr_push_credentials)
+                    )
+                creds[f"{AR_SECRET_PREFIX}{project_data.name}"] = (
+                    self._decode_push_secret(project_data.artifact_push_credentials)
+                )
+        return creds
+
+
+def run(dry_run: bool) -> None:
+    with QuayMirror(dry_run) as gcp_image_mirror:
+        gcp_image_mirror.run()

reconcile/gitlab_housekeeping.py

@@ -229,7 +229,7 @@ def verify_on_demand_tests(
             commit.id,
         ])
         if not dry_run and state_change:
-            markdown_report = f"On-demand Tests: \n\n All necessary tests have paased for latest [commit]({commit.web_url})\n"
+            markdown_report = f"On-demand Tests: \n\n All necessary tests have passed for latest [commit]({commit.web_url})\n"
             gl.delete_merge_request_comments(mr, startswith="On-demand Tests:")
             gl.add_comment_to_merge_request(mr, markdown_report)
             state.add(state_key, remaining_tests, force=True)

reconcile/gql_definitions/common/saas_files.py

@@ -18,6 +18,7 @@ from pydantic import (  # noqa: F401 # pylint: disable=W0611
 )
 
 from reconcile.gql_definitions.fragments.oc_connection_cluster import OcConnectionCluster
+from reconcile.gql_definitions.fragments.saas_slo_document import SLODocument
 from reconcile.gql_definitions.fragments.saas_target_namespace import SaasTargetNamespace
 from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
 
@@ -53,6 +54,50 @@ fragment OcConnectionCluster on Cluster_v1 {
   }
 }
 
+fragment SLODocument on SLODocument_v1 {
+    name
+    namespaces {
+      prometheusAccess {
+         url
+         username {
+         ... VaultSecret
+         }
+         password {
+           ... VaultSecret
+         }
+      }
+      namespace {
+        name
+        app {
+          name
+        }
+        cluster {
+          name
+          automationToken {
+          ... VaultSecret
+          }
+          prometheusUrl
+          spec {
+            private
+          }
+        }
+      }
+      SLONamespace {
+        name
+      }
+    }
+    slos {
+      name
+      expr
+      SLIType
+      SLOParameters {
+        window
+      }
+      SLOTarget
+      SLOTargetUnit
+    }
+}
+
 fragment SaasTargetNamespace on Namespace_v1 {
   name
   labels
@@ -253,6 +298,9 @@ query SaasFiles {
         namespace {
           ...SaasTargetNamespace
         }
+        slos {
+         ...SLODocument 
+        }
         namespaceSelector {
           jsonPathSelectors {
             include
@@ -512,6 +560,7 @@ class SaasResourceTemplateTargetV2(ConfiguredBaseModel):
     path: Optional[str] = Field(..., alias="path")
     name: Optional[str] = Field(..., alias="name")
     namespace: Optional[SaasTargetNamespace] = Field(..., alias="namespace")
+    slos: Optional[list[SLODocument]] = Field(..., alias="slos")
     namespace_selector: Optional[SaasResourceTemplateTargetNamespaceSelectorV1] = Field(..., alias="namespaceSelector")
     provider: Optional[str] = Field(..., alias="provider")
     ref: str = Field(..., alias="ref")

reconcile/gql_definitions/dashdotdb_slo/slo_documents_query.py

@@ -17,19 +17,11 @@ from pydantic import (  # noqa: F401 # pylint: disable=W0611
     Json,
 )
 
-from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+from reconcile.gql_definitions.fragments.saas_slo_document import SLODocument
 
 
 DEFINITION = """
-fragment VaultSecret on VaultSecret_v1 {
-    path
-    field
-    version
-    format
-}
-
-query SLODocuments {
-  slo_documents: slo_document_v1 {
+fragment SLODocument on SLODocument_v1 {
     name
     namespaces {
       prometheusAccess {
@@ -71,6 +63,18 @@ query SLODocuments {
       SLOTarget
       SLOTargetUnit
     }
+}
+
+fragment VaultSecret on VaultSecret_v1 {
+    path
+    field
+    version
+    format
+}
+
+query SLODocuments {
+  slo_documents: slo_document_v1 {
+  ... SLODocument
   }
 }
 """
@@ -82,64 +86,8 @@ class ConfiguredBaseModel(BaseModel):
         extra=Extra.forbid
 
 
-class SLOExternalPrometheusAccessV1(ConfiguredBaseModel):
-    url: str = Field(..., alias="url")
-    username: Optional[VaultSecret] = Field(..., alias="username")
-    password: Optional[VaultSecret] = Field(..., alias="password")
-
-
-class AppV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-
-
-class ClusterSpecV1(ConfiguredBaseModel):
-    private: bool = Field(..., alias="private")
-
-
-class ClusterV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-    automation_token: Optional[VaultSecret] = Field(..., alias="automationToken")
-    prometheus_url: str = Field(..., alias="prometheusUrl")
-    spec: Optional[ClusterSpecV1] = Field(..., alias="spec")
-
-
-class NamespaceV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-    app: AppV1 = Field(..., alias="app")
-    cluster: ClusterV1 = Field(..., alias="cluster")
-
-
-class SLONamespacesV1_NamespaceV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-
-
-class SLONamespacesV1(ConfiguredBaseModel):
-    prometheus_access: Optional[SLOExternalPrometheusAccessV1] = Field(..., alias="prometheusAccess")
-    namespace: NamespaceV1 = Field(..., alias="namespace")
-    slo_namespace: Optional[SLONamespacesV1_NamespaceV1] = Field(..., alias="SLONamespace")
-
-
-class SLODocumentSLOSLOParametersV1(ConfiguredBaseModel):
-    window: str = Field(..., alias="window")
-
-
-class SLODocumentSLOV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-    expr: str = Field(..., alias="expr")
-    sli_type: str = Field(..., alias="SLIType")
-    slo_parameters: SLODocumentSLOSLOParametersV1 = Field(..., alias="SLOParameters")
-    slo_target: float = Field(..., alias="SLOTarget")
-    slo_target_unit: str = Field(..., alias="SLOTargetUnit")
-
-
-class SLODocumentV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-    namespaces: list[SLONamespacesV1] = Field(..., alias="namespaces")
-    slos: Optional[list[SLODocumentSLOV1]] = Field(..., alias="slos")
-
-
 class SLODocumentsQueryData(ConfiguredBaseModel):
-    slo_documents: Optional[list[SLODocumentV1]] = Field(..., alias="slo_documents")
+    slo_documents: Optional[list[SLODocument]] = Field(..., alias="slo_documents")
 
 
 def query(query_func: Callable, **kwargs: Any) -> SLODocumentsQueryData:

reconcile/gql_definitions/fragments/container_image_mirror.py

@@ -0,0 +1,33 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class ContainerImageMirror(ConfiguredBaseModel):
+    url: str = Field(..., alias="url")
+    pull_credentials: Optional[VaultSecret] = Field(..., alias="pullCredentials")
+    tags: Optional[list[str]] = Field(..., alias="tags")
+    tags_exclude: Optional[list[str]] = Field(..., alias="tagsExclude")

reconcile/gql_definitions/fragments/saas_slo_document.py

@@ -0,0 +1,82 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class SLOExternalPrometheusAccessV1(ConfiguredBaseModel):
+    url: str = Field(..., alias="url")
+    username: Optional[VaultSecret] = Field(..., alias="username")
+    password: Optional[VaultSecret] = Field(..., alias="password")
+
+
+class AppV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+
+
+class ClusterSpecV1(ConfiguredBaseModel):
+    private: bool = Field(..., alias="private")
+
+
+class ClusterV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    automation_token: Optional[VaultSecret] = Field(..., alias="automationToken")
+    prometheus_url: str = Field(..., alias="prometheusUrl")
+    spec: Optional[ClusterSpecV1] = Field(..., alias="spec")
+
+
+class NamespaceV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    app: AppV1 = Field(..., alias="app")
+    cluster: ClusterV1 = Field(..., alias="cluster")
+
+
+class SLONamespacesV1_NamespaceV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+
+
+class SLONamespacesV1(ConfiguredBaseModel):
+    prometheus_access: Optional[SLOExternalPrometheusAccessV1] = Field(..., alias="prometheusAccess")
+    namespace: NamespaceV1 = Field(..., alias="namespace")
+    slo_namespace: Optional[SLONamespacesV1_NamespaceV1] = Field(..., alias="SLONamespace")
+
+
+class SLODocumentSLOSLOParametersV1(ConfiguredBaseModel):
+    window: str = Field(..., alias="window")
+
+
+class SLODocumentSLOV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    expr: str = Field(..., alias="expr")
+    sli_type: str = Field(..., alias="SLIType")
+    slo_parameters: SLODocumentSLOSLOParametersV1 = Field(..., alias="SLOParameters")
+    slo_target: float = Field(..., alias="SLOTarget")
+    slo_target_unit: str = Field(..., alias="SLOTargetUnit")
+
+
+class SLODocument(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    namespaces: list[SLONamespacesV1] = Field(..., alias="namespaces")
+    slos: Optional[list[SLODocumentSLOV1]] = Field(..., alias="slos")

reconcile/gql_definitions/gcp/gcp_docker_repos.py

@@ -0,0 +1,128 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.container_image_mirror import ContainerImageMirror
+
+
+DEFINITION = """
+fragment ContainerImageMirror on ContainerImageMirror_v1 {
+  url
+  pullCredentials {
+    ...VaultSecret
+  }
+  tags
+  tagsExclude
+}
+
+fragment VaultSecret on VaultSecret_v1 {
+    path
+    field
+    version
+    format
+}
+
+query GcpDockerRepos {
+  apps: apps_v1 {
+    gcrRepos {
+      project {
+        name
+      }
+      items {
+        name
+        mirror {
+          ...ContainerImageMirror
+        }
+      }
+    }
+    artifactRegistryMirrors {
+      project {
+        name
+      }
+      items {
+        imageURL
+        mirror {
+          ...ContainerImageMirror
+        }
+      }
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class GcpProjectV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+
+
+class AppGcrReposItemsV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    mirror: Optional[ContainerImageMirror] = Field(..., alias="mirror")
+
+
+class AppGcrReposV1(ConfiguredBaseModel):
+    project: GcpProjectV1 = Field(..., alias="project")
+    items: list[AppGcrReposItemsV1] = Field(..., alias="items")
+
+
+class AppArtifactRegistryMirrorsV1_GcpProjectV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+
+
+class AppArtifactRegistryMirrorsItemsV1(ConfiguredBaseModel):
+    image_url: str = Field(..., alias="imageURL")
+    mirror: ContainerImageMirror = Field(..., alias="mirror")
+
+
+class AppArtifactRegistryMirrorsV1(ConfiguredBaseModel):
+    project: AppArtifactRegistryMirrorsV1_GcpProjectV1 = Field(..., alias="project")
+    items: list[AppArtifactRegistryMirrorsItemsV1] = Field(..., alias="items")
+
+
+class AppV1(ConfiguredBaseModel):
+    gcr_repos: Optional[list[AppGcrReposV1]] = Field(..., alias="gcrRepos")
+    artifact_registry_mirrors: Optional[list[AppArtifactRegistryMirrorsV1]] = Field(..., alias="artifactRegistryMirrors")
+
+
+class GcpDockerReposQueryData(ConfiguredBaseModel):
+    apps: Optional[list[AppV1]] = Field(..., alias="apps")
+
+
+def query(query_func: Callable, **kwargs: Any) -> GcpDockerReposQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        GcpDockerReposQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return GcpDockerReposQueryData(**raw_data)