qontract-reconcile 0.10.2.dev159__py3-none-any.whl → 0.10.2.dev161__py3-none-any.whl

reconcile/gql_definitions/fragments/container_image_mirror.py

@@ -0,0 +1,33 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class ContainerImageMirror(ConfiguredBaseModel):
+    url: str = Field(..., alias="url")
+    pull_credentials: Optional[VaultSecret] = Field(..., alias="pullCredentials")
+    tags: Optional[list[str]] = Field(..., alias="tags")
+    tags_exclude: Optional[list[str]] = Field(..., alias="tagsExclude")

reconcile/gql_definitions/gcp/gcp_docker_repos.py

@@ -0,0 +1,128 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.container_image_mirror import ContainerImageMirror
+
+
+DEFINITION = """
+fragment ContainerImageMirror on ContainerImageMirror_v1 {
+  url
+  pullCredentials {
+    ...VaultSecret
+  }
+  tags
+  tagsExclude
+}
+
+fragment VaultSecret on VaultSecret_v1 {
+    path
+    field
+    version
+    format
+}
+
+query GcpDockerRepos {
+  apps: apps_v1 {
+    gcrRepos {
+      project {
+        name
+      }
+      items {
+        name
+        mirror {
+          ...ContainerImageMirror
+        }
+      }
+    }
+    artifactRegistryMirrors {
+      project {
+        name
+      }
+      items {
+        imageURL
+        mirror {
+          ...ContainerImageMirror
+        }
+      }
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class GcpProjectV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+
+
+class AppGcrReposItemsV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    mirror: Optional[ContainerImageMirror] = Field(..., alias="mirror")
+
+
+class AppGcrReposV1(ConfiguredBaseModel):
+    project: GcpProjectV1 = Field(..., alias="project")
+    items: list[AppGcrReposItemsV1] = Field(..., alias="items")
+
+
+class AppArtifactRegistryMirrorsV1_GcpProjectV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+
+
+class AppArtifactRegistryMirrorsItemsV1(ConfiguredBaseModel):
+    image_url: str = Field(..., alias="imageURL")
+    mirror: ContainerImageMirror = Field(..., alias="mirror")
+
+
+class AppArtifactRegistryMirrorsV1(ConfiguredBaseModel):
+    project: AppArtifactRegistryMirrorsV1_GcpProjectV1 = Field(..., alias="project")
+    items: list[AppArtifactRegistryMirrorsItemsV1] = Field(..., alias="items")
+
+
+class AppV1(ConfiguredBaseModel):
+    gcr_repos: Optional[list[AppGcrReposV1]] = Field(..., alias="gcrRepos")
+    artifact_registry_mirrors: Optional[list[AppArtifactRegistryMirrorsV1]] = Field(..., alias="artifactRegistryMirrors")
+
+
+class GcpDockerReposQueryData(ConfiguredBaseModel):
+    apps: Optional[list[AppV1]] = Field(..., alias="apps")
+
+
+def query(query_func: Callable, **kwargs: Any) -> GcpDockerReposQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        GcpDockerReposQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return GcpDockerReposQueryData(**raw_data)

reconcile/gql_definitions/gcp/gcp_projects.py

@@ -0,0 +1,77 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+
+
+DEFINITION = """
+fragment VaultSecret on VaultSecret_v1 {
+    path
+    field
+    version
+    format
+}
+
+query GcpProjects {
+  gcp_projects: gcp_projects_v1 {
+    name
+    gcrPushCredentials {
+      ...VaultSecret
+    }
+    artifactPushCredentials {
+      ...VaultSecret
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class GcpProjectV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    gcr_push_credentials: Optional[VaultSecret] = Field(..., alias="gcrPushCredentials")
+    artifact_push_credentials: VaultSecret = Field(..., alias="artifactPushCredentials")
+
+
+class GcpProjectsQueryData(ConfiguredBaseModel):
+    gcp_projects: Optional[list[GcpProjectV1]] = Field(..., alias="gcp_projects")
+
+
+def query(query_func: Callable, **kwargs: Any) -> GcpProjectsQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        GcpProjectsQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return GcpProjectsQueryData(**raw_data)

reconcile/gql_definitions/introspection.json

@@ -15959,6 +15959,26 @@
                             "isDeprecated": false,
                             "deprecationReason": null
                         },
+                        {
+                            "name": "artifactRegistryMirrors",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "LIST",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "NON_NULL",
+                                    "name": null,
+                                    "ofType": {
+                                        "kind": "OBJECT",
+                                        "name": "AppArtifactRegistryMirrors_v1",
+                                        "ofType": null
+                                    }
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
                         {
                             "name": "quayRepos",
                             "description": null,
@@ -16816,7 +16836,7 @@
                             "deprecationReason": null
                         },
                         {
-                            "name": "pushCredentials",
+                            "name": "gcrPushCredentials",
                             "description": null,
                             "args": [],
                             "type": {
@@ -16826,6 +16846,22 @@
                             },
                             "isDeprecated": false,
                             "deprecationReason": null
+                        },
+                        {
+                            "name": "artifactPushCredentials",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "NON_NULL",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "OBJECT",
+                                    "name": "VaultSecret_v1",
+                                    "ofType": null
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
                         }
                     ],
                     "inputFields": null,
@@ -17027,6 +17063,100 @@
                     "enumValues": null,
                     "possibleTypes": null
                 },
+                {
+                    "kind": "OBJECT",
+                    "name": "AppArtifactRegistryMirrors_v1",
+                    "description": null,
+                    "fields": [
+                        {
+                            "name": "project",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "NON_NULL",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "OBJECT",
+                                    "name": "GcpProject_v1",
+                                    "ofType": null
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "items",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "NON_NULL",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "LIST",
+                                    "name": null,
+                                    "ofType": {
+                                        "kind": "NON_NULL",
+                                        "name": null,
+                                        "ofType": {
+                                            "kind": "OBJECT",
+                                            "name": "AppArtifactRegistryMirrorsItems_v1",
+                                            "ofType": null
+                                        }
+                                    }
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        }
+                    ],
+                    "inputFields": null,
+                    "interfaces": [],
+                    "enumValues": null,
+                    "possibleTypes": null
+                },
+                {
+                    "kind": "OBJECT",
+                    "name": "AppArtifactRegistryMirrorsItems_v1",
+                    "description": null,
+                    "fields": [
+                        {
+                            "name": "imageURL",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "NON_NULL",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "SCALAR",
+                                    "name": "String",
+                                    "ofType": null
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "mirror",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "NON_NULL",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "OBJECT",
+                                    "name": "ContainerImageMirror_v1",
+                                    "ofType": null
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        }
+                    ],
+                    "inputFields": null,
+                    "interfaces": [],
+                    "enumValues": null,
+                    "possibleTypes": null
+                },
                 {
                     "kind": "OBJECT",
                     "name": "AppQuayRepos_v1",

reconcile/typed_queries/aws_cloudwatch_log_retention/aws_accounts.py

@@ -0,0 +1,12 @@
+from reconcile.gql_definitions.aws_cloudwatch_log_retention.aws_accounts import (
+    AWSAccountV1,
+    query,
+)
+from reconcile.utils.gql import GqlApi
+
+
+def get_aws_accounts(
+    gql_api: GqlApi,
+) -> list[AWSAccountV1]:
+    data = query(query_func=gql_api.query)
+    return data.accounts or []

reconcile/gcr_mirror.py

@@ -1,278 +0,0 @@
-import base64
-import logging
-import os
-import re
-import tempfile
-import time
-from collections import defaultdict
-from typing import Any, Self
-
-import requests
-from sretoolbox.container import (
-    Image,
-    Skopeo,
-)
-from sretoolbox.container.image import ImageComparisonError
-from sretoolbox.container.skopeo import SkopeoCmdError
-
-from reconcile import queries
-from reconcile.utils import gql
-from reconcile.utils.secret_reader import SecretReader
-
-_LOG = logging.getLogger(__name__)
-
-QONTRACT_INTEGRATION = "gcr-mirror"
-REQUEST_TIMEOUT = 60
-
-
-class QuayMirror:
-    GCR_PROJECT_CATALOG_QUERY = """
-    {
-      projects: gcp_projects_v1 {
-        name
-        pushCredentials {
-          path
-          field
-          version
-          format
-        }
-      }
-    }
-    """
-
-    GCR_REPOS_QUERY = """
-    {
-      apps: apps_v1 {
-        gcrRepos {
-          project {
-            name
-          }
-          items {
-            name
-            mirror {
-              url
-              pullCredentials {
-                path
-                field
-                version
-                format
-              }
-              tags
-              tagsExclude
-            }
-          }
-        }
-      }
-    }
-    """
-
-    def __init__(self, dry_run: bool = False) -> None:
-        self.dry_run = dry_run
-        self.gqlapi = gql.get_api()
-        settings = queries.get_app_interface_settings()
-        self.secret_reader = SecretReader(settings=settings)
-        self.skopeo_cli = Skopeo(dry_run)
-        self.push_creds = self._get_push_creds()
-        self.session = requests.Session()
-
-    def __enter__(self) -> Self:
-        return self
-
-    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
-        self.session.close()
-
-    def run(self) -> None:
-        sync_tasks = self.process_sync_tasks()
-        for org, data in sync_tasks.items():
-            for item in data:
-                try:
-                    self.skopeo_cli.copy(
-                        src_image=item["mirror_url"],
-                        src_creds=item["mirror_creds"],
-                        dst_image=item["image_url"],
-                        dest_creds=self.push_creds[org],
-                    )
-                except SkopeoCmdError as details:
-                    _LOG.error("[%s]", details)
-
-    def process_repos_query(self) -> dict[str, list[dict[str, Any]]]:
-        result = self.gqlapi.query(self.GCR_REPOS_QUERY)
-
-        summary = defaultdict(list)
-
-        for app in result["apps"]:
-            gcr_repos = app.get("gcrRepos")
-
-            if gcr_repos is None:
-                continue
-
-            for gcr_repo in gcr_repos:
-                project = gcr_repo["project"]["name"]
-                server_url = gcr_repo["project"].get("serverUrl") or "gcr.io"
-                for item in gcr_repo["items"]:
-                    if item["mirror"] is None:
-                        continue
-
-                    summary[project].append({
-                        "name": item["name"],
-                        "mirror": item["mirror"],
-                        "server_url": server_url,
-                    })
-
-        return summary
-
-    @staticmethod
-    def sync_tag(
-        tags: list[str] | None, tags_exclude: list[str] | None, candidate: str
-    ) -> bool:
-        if tags is not None:
-            # When tags is defined, we don't look at tags_exclude
-            return any(re.match(tag, candidate) for tag in tags)
-
-        if tags_exclude is not None:
-            for tag_exclude in tags_exclude:
-                if re.match(tag_exclude, candidate):
-                    return False
-            return True
-
-        # Both tags and tags_exclude are None, so
-        # tag must be synced
-        return True
-
-    def process_sync_tasks(self) -> dict[str, list[dict[str, Any]]]:
-        eight_hours = 28800  # 60 * 60 * 8
-        is_deep_sync = self._is_deep_sync(interval=eight_hours)
-
-        summary = self.process_repos_query()
-
-        sync_tasks = defaultdict(list)
-        for org, data in summary.items():
-            for item in data:
-                image = Image(
-                    f"{item['server_url']}/{org}/{item['name']}",
-                    session=self.session,
-                    timeout=REQUEST_TIMEOUT,
-                )
-
-                mirror_url = item["mirror"]["url"]
-
-                username = None
-                password = None
-                mirror_creds = None
-                if item["mirror"]["pullCredentials"] is not None:
-                    pull_credentials = item["mirror"]["pullCredentials"]
-                    raw_data = self.secret_reader.read_all(pull_credentials)
-                    username = raw_data["user"]
-                    password = raw_data["token"]
-                    mirror_creds = f"{username}:{password}"
-
-                image_mirror = Image(
-                    mirror_url,
-                    username=username,
-                    password=password,
-                    session=self.session,
-                    timeout=REQUEST_TIMEOUT,
-                )
-
-                tags = item["mirror"].get("tags")
-                tags_exclude = item["mirror"].get("tagsExclude")
-
-                for tag in image_mirror:
-                    if not self.sync_tag(
-                        tags=tags, tags_exclude=tags_exclude, candidate=tag
-                    ):
-                        continue
-
-                    upstream = image_mirror[tag]
-                    downstream = image[tag]
-                    if tag not in image:
-                        _LOG.debug(
-                            "Image %s and mirror %s are out off sync",
-                            downstream,
-                            upstream,
-                        )
-                        sync_tasks[org].append({
-                            "mirror_url": str(upstream),
-                            "mirror_creds": mirror_creds,
-                            "image_url": str(downstream),
-                        })
-                        continue
-
-                    # Deep (slow) check only in non dry-run mode
-                    if self.dry_run:
-                        _LOG.debug(
-                            "Image %s and mirror %s are in sync", downstream, upstream
-                        )
-                        continue
-
-                    # Deep (slow) check only from time to time
-                    if not is_deep_sync:
-                        _LOG.debug(
-                            "Image %s and mirror %s are in sync", downstream, upstream
-                        )
-                        continue
-
-                    try:
-                        if downstream == upstream:
-                            _LOG.debug(
-                                "Image %s and mirror %s are in sync",
-                                downstream,
-                                upstream,
-                            )
-                            continue
-                    except ImageComparisonError as details:
-                        _LOG.error("[%s]", details)
-                        continue
-
-                    _LOG.debug(
-                        "Image %s and mirror %s are out of sync", downstream, upstream
-                    )
-                    sync_tasks[org].append({
-                        "mirror_url": str(upstream),
-                        "mirror_creds": mirror_creds,
-                        "image_url": str(downstream),
-                    })
-
-        return sync_tasks
-
-    def _is_deep_sync(self, interval: int) -> bool:
-        control_file_name = "qontract-reconcile-gcr-mirror.timestamp"
-        control_file_path = os.path.join(tempfile.gettempdir(), control_file_name)
-        try:
-            with open(control_file_path, encoding="locale") as file_obj:
-                last_deep_sync = float(file_obj.read())
-        except FileNotFoundError:
-            self._record_timestamp(control_file_path)
-            return True
-
-        next_deep_sync = last_deep_sync + interval
-        if time.time() >= next_deep_sync:
-            self._record_timestamp(control_file_path)
-            return True
-
-        return False
-
-    @staticmethod
-    def _record_timestamp(path: str) -> None:
-        with open(path, "w", encoding="locale") as file_object:
-            file_object.write(str(time.time()))
-
-    def _get_push_creds(self) -> dict[str, str]:
-        result = self.gqlapi.query(self.GCR_PROJECT_CATALOG_QUERY)
-
-        creds = {}
-        for project_data in result["projects"]:
-            push_secret = project_data["pushCredentials"]
-            if push_secret is None:
-                continue
-
-            raw_data = self.secret_reader.read_all(push_secret)
-            project = project_data["name"]
-            token = base64.b64decode(raw_data["token"]).decode()
-            creds[project] = f"{raw_data['user']}:{token}"
-        return creds
-
-
-def run(dry_run: bool) -> None:
-    with QuayMirror(dry_run) as gcr_mirror:
-        gcr_mirror.run()