qontract-reconcile 0.10.2.dev160__py3-none-any.whl → 0.10.2.dev173__py3-none-any.whl

tools/cli_commands/erv2.py

@@ -123,6 +123,7 @@ class Erv2Cli:
                 m_inventory.get_from_spec(spec), spec, self._er_settings
             )
         )
+        self.module_type = m_inventory.get_from_spec(spec).module_type
         self._resource = f.create_external_resource(spec, self._module_configuration)
         f.validate_external_resource(self._resource, self._module_configuration)
 
@@ -221,6 +222,66 @@ class Erv2Cli:
                 print(e.stdout.decode("utf-8"))
             raise
 
+    def build_terraform(self, credentials: Path) -> None:
+        input_file = self.temp / "input.json"
+        input_file.write_text(self.input_data)
+        tf_module = "tfmodule"
+
+        # delete previous ERv2 container
+        run(["docker", "rm", "-f", "erv2"], capture_output=True, check=True)
+
+        try:
+            with task(self.progress_spinner, "-- Running terraform"):
+                run(["docker", "pull", self.image], check=True)
+                run(
+                    [
+                        "docker",
+                        "run",
+                        "--name",
+                        "erv2",
+                        "-v",
+                        f"{input_file!s}:/inputs/input.json:Z",
+                        "-v",
+                        f"{credentials!s}:/credentials:Z",
+                        "-v",
+                        f"{self.temp}:/work:Z",
+                        "-e",
+                        "AWS_SHARED_CREDENTIALS_FILE=/credentials",
+                        "-e",
+                        f"TERRAFORM_MODULE_WORK_DIR=/tmp/{tf_module}",
+                        "-e",
+                        "LOCAL_STATE=False",
+                        self.image,
+                    ],
+                    check=True,
+                    capture_output=True,
+                )
+
+            with task(self.progress_spinner, "-- Copying the terraform module"):
+                run(
+                    [
+                        "docker",
+                        "cp",
+                        f"erv2:/tmp/{tf_module}",
+                        str(self.temp),
+                    ],
+                    check=True,
+                    capture_output=True,
+                )
+
+            # move all assets to local workdir
+            src_dir = self.temp / tf_module
+            for item in src_dir.iterdir():
+                if item.name != ".terraform":
+                    item.rename(self.temp / item.name)
+
+        except CalledProcessError as e:
+            if e.stderr:
+                print(e.stderr.decode("utf-8"))
+            if e.stdout:
+                print(e.stdout.decode("utf-8"))
+            raise
+
     def enter_shell(self, credentials: Path) -> None:
         """Run the ERv2 container and enter the shell."""
         input_file = self.temp / "input.json"

tools/qontract_cli.py

@@ -798,6 +798,11 @@ def ocm_addon_upgrade_policies(ctx: click.core.Context) -> None:
 
 
 @get.command()
+@click.option(
+    "--channel",
+    help="Specifies the channel that alerts stores",
+    type=str,
+)
 @click.option(
     "--days",
     help="Days to consider for the report. Cannot be used with timestamp options.",
@@ -816,13 +821,14 @@ def ocm_addon_upgrade_policies(ctx: click.core.Context) -> None:
     type=int,
 )
 @click.pass_context
-def sd_app_sre_alert_report(
+def alert_report(
     ctx: click.core.Context,
+    channel: str | None,
     days: int | None,
     from_timestamp: int | None,
     to_timestamp: int | None,
 ) -> None:
-    import tools.sd_app_sre_alert_report as report
+    import tools.alert_report as report
 
     if days:
         if from_timestamp or to_timestamp:
@@ -845,7 +851,9 @@ def sd_app_sre_alert_report(
             sys.exit(1)
 
     slack = slackapi_from_queries(
-        integration_name=report.QONTRACT_INTEGRATION, init_usergroups=False
+        integration_name=report.QONTRACT_INTEGRATION,
+        init_usergroups=False,
+        channel=channel,
     )
     alerts = report.group_alerts(
         slack.get_flat_conversation_history(
@@ -4381,8 +4389,10 @@ def migrate(ctx: click.Context, dry_run: bool, skip_build: bool) -> None:
 
         with task(progress, "(erv2) Building the terraform configuration"):
             if not skip_build:
-                # build the CDKTF output
-                erv2cli.build_cdktf(credentials_file)
+                if erv2cli.module_type == "cdktf":
+                    erv2cli.build_cdktf(credentials_file)
+                else:
+                    erv2cli.build_terraform(credentials_file)
             erv2_tf_cli = TerraformCli(
                 temp_erv2, dry_run=dry_run, progress_spinner=progress
             )

reconcile/gcr_mirror.py

@@ -1,278 +0,0 @@
-import base64
-import logging
-import os
-import re
-import tempfile
-import time
-from collections import defaultdict
-from typing import Any, Self
-
-import requests
-from sretoolbox.container import (
-    Image,
-    Skopeo,
-)
-from sretoolbox.container.image import ImageComparisonError
-from sretoolbox.container.skopeo import SkopeoCmdError
-
-from reconcile import queries
-from reconcile.utils import gql
-from reconcile.utils.secret_reader import SecretReader
-
-_LOG = logging.getLogger(__name__)
-
-QONTRACT_INTEGRATION = "gcr-mirror"
-REQUEST_TIMEOUT = 60
-
-
-class QuayMirror:
-    GCR_PROJECT_CATALOG_QUERY = """
-    {
-      projects: gcp_projects_v1 {
-        name
-        pushCredentials {
-          path
-          field
-          version
-          format
-        }
-      }
-    }
-    """
-
-    GCR_REPOS_QUERY = """
-    {
-      apps: apps_v1 {
-        gcrRepos {
-          project {
-            name
-          }
-          items {
-            name
-            mirror {
-              url
-              pullCredentials {
-                path
-                field
-                version
-                format
-              }
-              tags
-              tagsExclude
-            }
-          }
-        }
-      }
-    }
-    """
-
-    def __init__(self, dry_run: bool = False) -> None:
-        self.dry_run = dry_run
-        self.gqlapi = gql.get_api()
-        settings = queries.get_app_interface_settings()
-        self.secret_reader = SecretReader(settings=settings)
-        self.skopeo_cli = Skopeo(dry_run)
-        self.push_creds = self._get_push_creds()
-        self.session = requests.Session()
-
-    def __enter__(self) -> Self:
-        return self
-
-    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
-        self.session.close()
-
-    def run(self) -> None:
-        sync_tasks = self.process_sync_tasks()
-        for org, data in sync_tasks.items():
-            for item in data:
-                try:
-                    self.skopeo_cli.copy(
-                        src_image=item["mirror_url"],
-                        src_creds=item["mirror_creds"],
-                        dst_image=item["image_url"],
-                        dest_creds=self.push_creds[org],
-                    )
-                except SkopeoCmdError as details:
-                    _LOG.error("[%s]", details)
-
-    def process_repos_query(self) -> dict[str, list[dict[str, Any]]]:
-        result = self.gqlapi.query(self.GCR_REPOS_QUERY)
-
-        summary = defaultdict(list)
-
-        for app in result["apps"]:
-            gcr_repos = app.get("gcrRepos")
-
-            if gcr_repos is None:
-                continue
-
-            for gcr_repo in gcr_repos:
-                project = gcr_repo["project"]["name"]
-                server_url = gcr_repo["project"].get("serverUrl") or "gcr.io"
-                for item in gcr_repo["items"]:
-                    if item["mirror"] is None:
-                        continue
-
-                    summary[project].append({
-                        "name": item["name"],
-                        "mirror": item["mirror"],
-                        "server_url": server_url,
-                    })
-
-        return summary
-
-    @staticmethod
-    def sync_tag(
-        tags: list[str] | None, tags_exclude: list[str] | None, candidate: str
-    ) -> bool:
-        if tags is not None:
-            # When tags is defined, we don't look at tags_exclude
-            return any(re.match(tag, candidate) for tag in tags)
-
-        if tags_exclude is not None:
-            for tag_exclude in tags_exclude:
-                if re.match(tag_exclude, candidate):
-                    return False
-            return True
-
-        # Both tags and tags_exclude are None, so
-        # tag must be synced
-        return True
-
-    def process_sync_tasks(self) -> dict[str, list[dict[str, Any]]]:
-        eight_hours = 28800  # 60 * 60 * 8
-        is_deep_sync = self._is_deep_sync(interval=eight_hours)
-
-        summary = self.process_repos_query()
-
-        sync_tasks = defaultdict(list)
-        for org, data in summary.items():
-            for item in data:
-                image = Image(
-                    f"{item['server_url']}/{org}/{item['name']}",
-                    session=self.session,
-                    timeout=REQUEST_TIMEOUT,
-                )
-
-                mirror_url = item["mirror"]["url"]
-
-                username = None
-                password = None
-                mirror_creds = None
-                if item["mirror"]["pullCredentials"] is not None:
-                    pull_credentials = item["mirror"]["pullCredentials"]
-                    raw_data = self.secret_reader.read_all(pull_credentials)
-                    username = raw_data["user"]
-                    password = raw_data["token"]
-                    mirror_creds = f"{username}:{password}"
-
-                image_mirror = Image(
-                    mirror_url,
-                    username=username,
-                    password=password,
-                    session=self.session,
-                    timeout=REQUEST_TIMEOUT,
-                )
-
-                tags = item["mirror"].get("tags")
-                tags_exclude = item["mirror"].get("tagsExclude")
-
-                for tag in image_mirror:
-                    if not self.sync_tag(
-                        tags=tags, tags_exclude=tags_exclude, candidate=tag
-                    ):
-                        continue
-
-                    upstream = image_mirror[tag]
-                    downstream = image[tag]
-                    if tag not in image:
-                        _LOG.debug(
-                            "Image %s and mirror %s are out off sync",
-                            downstream,
-                            upstream,
-                        )
-                        sync_tasks[org].append({
-                            "mirror_url": str(upstream),
-                            "mirror_creds": mirror_creds,
-                            "image_url": str(downstream),
-                        })
-                        continue
-
-                    # Deep (slow) check only in non dry-run mode
-                    if self.dry_run:
-                        _LOG.debug(
-                            "Image %s and mirror %s are in sync", downstream, upstream
-                        )
-                        continue
-
-                    # Deep (slow) check only from time to time
-                    if not is_deep_sync:
-                        _LOG.debug(
-                            "Image %s and mirror %s are in sync", downstream, upstream
-                        )
-                        continue
-
-                    try:
-                        if downstream == upstream:
-                            _LOG.debug(
-                                "Image %s and mirror %s are in sync",
-                                downstream,
-                                upstream,
-                            )
-                            continue
-                    except ImageComparisonError as details:
-                        _LOG.error("[%s]", details)
-                        continue
-
-                    _LOG.debug(
-                        "Image %s and mirror %s are out of sync", downstream, upstream
-                    )
-                    sync_tasks[org].append({
-                        "mirror_url": str(upstream),
-                        "mirror_creds": mirror_creds,
-                        "image_url": str(downstream),
-                    })
-
-        return sync_tasks
-
-    def _is_deep_sync(self, interval: int) -> bool:
-        control_file_name = "qontract-reconcile-gcr-mirror.timestamp"
-        control_file_path = os.path.join(tempfile.gettempdir(), control_file_name)
-        try:
-            with open(control_file_path, encoding="locale") as file_obj:
-                last_deep_sync = float(file_obj.read())
-        except FileNotFoundError:
-            self._record_timestamp(control_file_path)
-            return True
-
-        next_deep_sync = last_deep_sync + interval
-        if time.time() >= next_deep_sync:
-            self._record_timestamp(control_file_path)
-            return True
-
-        return False
-
-    @staticmethod
-    def _record_timestamp(path: str) -> None:
-        with open(path, "w", encoding="locale") as file_object:
-            file_object.write(str(time.time()))
-
-    def _get_push_creds(self) -> dict[str, str]:
-        result = self.gqlapi.query(self.GCR_PROJECT_CATALOG_QUERY)
-
-        creds = {}
-        for project_data in result["projects"]:
-            push_secret = project_data["pushCredentials"]
-            if push_secret is None:
-                continue
-
-            raw_data = self.secret_reader.read_all(push_secret)
-            project = project_data["name"]
-            token = base64.b64decode(raw_data["token"]).decode()
-            creds[project] = f"{raw_data['user']}:{token}"
-        return creds
-
-
-def run(dry_run: bool) -> None:
-    with QuayMirror(dry_run) as gcr_mirror:
-        gcr_mirror.run()