qontract-reconcile 0.10.1rc1185__py3-none-any.whl → 0.10.1rc1187__py3-none-any.whl

{qontract_reconcile-0.10.1rc1185.dist-info → qontract_reconcile-0.10.1rc1187.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc1185
+Version: 0.10.1rc1187
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc1185.dist-info → qontract_reconcile-0.10.1rc1187.dist-info}/RECORD

@@ -838,12 +838,12 @@ tools/app_interface_metrics_exporter.py,sha256=zkwkxdAUAxjdc-pzx2_oJXG25fo0Fnyd5
 tools/app_interface_reporter.py,sha256=oZPib4HPq0aZ2Zui1QGJGk6qQdfpeihujGDBnSdKyGE,17627
 tools/glitchtip_access_reporter.py,sha256=oPBnk_YoDuljU3v0FaChzOwwnk4vap1xEE67QEjzdqs,2948
 tools/glitchtip_access_revalidation.py,sha256=8kbBJk04mkq28kWoRDDkfCGIF3GRg3pJrFAh1sW0dbk,2821
-tools/qontract_cli.py,sha256=KfmjfaCuyd8F68oY1hHd2tEf0PY0kLd6t8sWqR81nmc,142498
+tools/qontract_cli.py,sha256=EsrEPKyeVBeViz0x21hfA25vkaHVEKBxHRyhozhdXVY,143933
 tools/sd_app_sre_alert_report.py,sha256=e9vAdyenUz2f5c8-z-5WY0wv-SJ9aePKDH2r4IwB6pc,5063
 tools/template_validation.py,sha256=qpKYaTgk0GOPGa2Ct5_5sKdwIHtCAKIBGzsMPuJU5fw,3371
 tools/cli_commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tools/cli_commands/container_images_report.py,sha256=PCJIzvUqiYmTdn5xJFcxHocCYp6dprrsJ_lkYdl3ET8,4417
-tools/cli_commands/erv2.py,sha256=469qdhyaf7thpPQ4hJSurvmxBqYDJsoI8H4AigQIF7U,20737
+tools/cli_commands/container_images_report.py,sha256=64LXkv3eoWtMFZwBgkYE9jSnReDD7A9M0GEVXsR9aGk,5183
+tools/cli_commands/erv2.py,sha256=f_Zc5ZCPenXbFj8zv_kcKJgmGkErYHGNIhbzOfOaD4Q,22041
 tools/cli_commands/gpg_encrypt.py,sha256=x02JOMn834z89YSNvr5B-oJky7rR1C0begCkPh45eHk,4958
 tools/cli_commands/systems_and_tools.py,sha256=EMHOF1AtUDaoSk0bbjl6oUKYAz4rTZjIBaF-6E6GspM,16816
 tools/cli_commands/cost_report/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -877,13 +877,13 @@ tools/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tools/test/conftest.py,sha256=CsDbu4otrxb7X7kXKKGyV3ZEzu3pCkgjCoCGiHNx6zc,2401
 tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvftCWEEf-g1mfXOtgCog-g,1271
 tools/test/test_erv2.py,sha256=EAS7QuJkHisRVO9bMGxm662L5B6i66wF_mT9PAjVzrU,3128
-tools/test/test_get_container_images.py,sha256=L2XzfmYAd6WZ17UXNnr8Z4iwoGcCvQ0vN6gxAZ7gEws,6097
+tools/test/test_get_container_images.py,sha256=QUSb0PjZnL35_dhniMux1_D2G_2vvpcasr8tpwPLwIQ,7125
 tools/test/test_qontract_cli.py,sha256=iuzKbQ6ahinvjoQmQLBrG4shey0z-1rB6qCgS8T6dgU,5789
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc1185.dist-info/METADATA,sha256=QatFHWfiRw29TrdKQtJhxXlwWCTa6pwrNTKTnxBMJtY,2213
-qontract_reconcile-0.10.1rc1185.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-qontract_reconcile-0.10.1rc1185.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc1185.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc1185.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc1187.dist-info/METADATA,sha256=ELcUNPGIHkuAtwY7r9iKGDrX38eFrABpKVzusb48UB8,2213
+qontract_reconcile-0.10.1rc1187.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+qontract_reconcile-0.10.1rc1187.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc1187.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc1187.dist-info/RECORD,,

tools/cli_commands/container_images_report.py

@@ -20,7 +20,8 @@ IMAGE_NAME_REGEX = re.compile(r"^(?P<name>[a-zA-Z0-9][a-zA-Z0-9/_.-]+)(?:@sha256
 
 class NamespaceImages(BaseModel):
     namespace_name: str
-    image_names: list[str]
+    image_names: list[str] | None = None
+    error_message: str | None = None
 
 
 def get_all_pods_images(
@@ -77,9 +78,14 @@ def fetch_pods_images_from_namespaces(
         oc_map=oc_map,
     )
 
+    errors: defaultdict = defaultdict(int)
     result: defaultdict = defaultdict(_get_all_images_default)
     for ni in all_namespace_images:
-        for name in ni.image_names:
+        if ni.error_message:
+            errors[f"{ni.namespace_name}/{ni.error_message}"] += 1
+            continue
+
+        for name in ni.image_names or []:
             result[name]["namespaces"].add(ni.namespace_name)
             result[name]["count"] += 1
 
@@ -92,7 +98,7 @@ def fetch_pods_images_from_namespaces(
         include_pattern_compiled = re.compile(include_pattern)
 
     result_filtered_flattened: list[dict[str, Any]] = []
-    for name, value in result.items():
+    for name, value in sorted(result.items()):
         if include_pattern_compiled and not include_pattern_compiled.match(name):
             continue
         if exclude_pattern_compiled and exclude_pattern_compiled.match(name):
@@ -104,6 +110,16 @@ def fetch_pods_images_from_namespaces(
             "count": value["count"],
         })
 
+    # append errors if they exist in the filtered result
+    # not very canonical, but it is better than ignoring them
+    if errors:
+        for message, count in sorted(errors.items()):
+            result_filtered_flattened.append({
+                "name": "error",
+                "namespaces": message,
+                "count": count,
+            })
+
     return result_filtered_flattened
 
 
@@ -113,15 +129,22 @@ def _get_all_images_default() -> dict[str, Any]:
 
 def _get_namespace_images(ns: NamespaceV1, oc_map: OCMap) -> NamespaceImages:
     image_names = []
-    oc = oc_map.get_cluster(ns.cluster.name)
-    pod_items = oc.get_items("Pod", namespace=ns.name)
-    for pod in pod_items:
-        containers = pod.get("spec", {}).get("containers", [])
-        containers.extend(pod.get("spec", {}).get("initContainers", []))
-
-        for c in containers:
-            if m := IMAGE_NAME_REGEX.match(c["image"]):
-                image_names.append(m.group("name"))
+
+    try:
+        oc = oc_map.get_cluster(ns.cluster.name)
+        pod_items = oc.get_items("Pod", namespace=ns.name)
+        for pod in pod_items:
+            containers = pod.get("spec", {}).get("containers", [])
+            containers.extend(pod.get("spec", {}).get("initContainers", []))
+
+            for c in containers:
+                if m := IMAGE_NAME_REGEX.match(c["image"]):
+                    image_names.append(m.group("name"))
+    except Exception as exc:
+        return NamespaceImages(
+            namespace_name=ns.name,
+            error_message=str(exc),
+        )
 
     return NamespaceImages(
         namespace_name=ns.name,

tools/cli_commands/erv2.py

@@ -215,8 +215,44 @@ class Erv2Cli:
                     capture_output=True,
                 )
         except CalledProcessError as e:
-            print(e.stderr.decode("utf-8"))
-            print(e.stdout.decode("utf-8"))
+            if e.stderr:
+                print(e.stderr.decode("utf-8"))
+            if e.stdout:
+                print(e.stdout.decode("utf-8"))
+            raise
+
+    def enter_shell(self, credentials: Path) -> None:
+        """Run the CDKTF container and enter the shell."""
+        input_file = self.temp / "input.json"
+        input_file.write_text(self.input_data)
+
+        try:
+            run(["docker", "pull", self.image], check=True, capture_output=True)
+            run(
+                [
+                    "docker",
+                    "run",
+                    "--name",
+                    "erv2-debug-shell",
+                    "--rm",
+                    "-it",
+                    "--mount",
+                    f"type=bind,source={input_file!s},target=/inputs/input.json",
+                    "--mount",
+                    f"type=bind,source={credentials!s},target=/credentials",
+                    "-e",
+                    "AWS_SHARED_CREDENTIALS_FILE=/credentials",
+                    "--entrypoint",
+                    "/bin/bash",
+                    self.image,
+                ],
+                check=True,
+            )
+        except CalledProcessError as e:
+            if e.stderr:
+                print(e.stderr.decode("utf-8"))
+            if e.stdout:
+                print(e.stdout.decode("utf-8"))
             raise
 
 
@@ -236,8 +272,10 @@ def tf_run(path: Path, cmd: list[str]) -> str:
             env=env,
         ).stdout.decode("utf-8")
     except CalledProcessError as e:
-        print(e.stderr.decode("utf-8"))
-        print(e.stdout.decode("utf-8"))
+        if e.stderr:
+            print(e.stderr.decode("utf-8"))
+        if e.stdout:
+            print(e.stdout.decode("utf-8"))
         raise
 
 

tools/qontract_cli.py

@@ -7,6 +7,7 @@ import logging
 import os
 import re
 import sys
+import tempfile
 import textwrap
 from collections import defaultdict
 from datetime import (
@@ -4203,7 +4204,7 @@ def request_reconciliation(ctx):
 
 
 @external_resources.command()
-@binary(["terraform"])
+@binary(["terraform", "docker"])
 @binary_version("terraform", ["version"], TERRAFORM_VERSION_REGEX, TERRAFORM_VERSION)
 @click.option(
     "--dry-run/--no-dry-run",
@@ -4320,6 +4321,39 @@ def migrate(ctx, dry_run: bool, skip_build: bool) -> None:
     rich_print(f"[b red]Please remove the temporary directory ({tempdir}) manually!")
 
 
+@external_resources.command()
+@binary(["docker"])
+@click.pass_context
+def debug_shell(ctx) -> None:
+    """Enter an ERv2 debug shell to manually migrate resources."""
+    # use a temporary directory in $HOME. The MacOS colima default configuration allows docker mounts from $HOME.
+    with tempfile.TemporaryDirectory(dir=Path.home(), prefix="erv2-debug.") as _tempdir:
+        tempdir = Path(_tempdir)
+        with progress_spinner() as progress:
+            with task(progress, "Preparing environment ..."):
+                credentials_file = tempdir / "credentials"
+                credentials_file.write_text(
+                    ctx.obj["secret_reader"].read_with_parameters(
+                        path=f"app-sre/external-resources/{ctx.obj['provisioner']}",
+                        field="credentials",
+                        format=None,
+                        version=None,
+                    )
+                )
+            os.environ["AWS_SHARED_CREDENTIALS_FILE"] = str(credentials_file)
+
+        erv2cli = Erv2Cli(
+            provision_provider=ctx.obj["provision_provider"],
+            provisioner=ctx.obj["provisioner"],
+            provider=ctx.obj["provider"],
+            identifier=ctx.obj["identifier"],
+            secret_reader=ctx.obj["secret_reader"],
+            temp_dir=tempdir,
+            progress_spinner=progress,
+        )
+        erv2cli.enter_shell(credentials_file)
+
+
 @get.command(help="Get all container images in app-interface defined namespaces")
 @cluster_name
 @namespace_name

tools/test/test_get_container_images.py

@@ -99,27 +99,17 @@ def oc_map(mocker: MockerFixture, oc: OCNative) -> OCMap:
     return oc_map
 
 
-# convert a list of dicts into a set of tuples to use it in assertions
-def _to_set(list_of_dicts: list[dict]) -> set[tuple]:
-    return {tuple(d.items()) for d in list_of_dicts}
-
-
-def testfetch_no_filter(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
+def test_fetch_no_filter(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
     images = fetch_pods_images_from_namespaces(
         namespaces=namespaces,
         oc_map=oc_map,
         thread_pool_size=2,
     )
 
-    assert _to_set(images) == _to_set([
-        {
-            "name": "quay.io/prometheus/blackbox-exporter",
-            "namespaces": "app-sre-observability-stage",
-            "count": 1,
-        },
+    assert images == [
         {
-            "name": "quay.io/redhat-services-prod/app-sre-tenant/gitlab-project-exporter-main/gitlab-project-exporter-main",
-            "namespaces": "app-sre-observability-stage",
+            "name": "quay.io/app-sre/clamav",
+            "namespaces": "app-sre-pipelines",
             "count": 1,
         },
         {
@@ -128,8 +118,8 @@ def testfetch_no_filter(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
             "count": 3,
         },
         {
-            "name": "quay.io/app-sre/clamav",
-            "namespaces": "app-sre-pipelines",
+            "name": "quay.io/prometheus/blackbox-exporter",
+            "namespaces": "app-sre-observability-stage",
             "count": 1,
         },
         {
@@ -138,26 +128,31 @@ def testfetch_no_filter(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
             "count": 1,
         },
         {
-            "name": "registry.redhat.io/openshift-pipelines/pipelines-entrypoint-rhel8",
-            "namespaces": "app-sre-pipelines",
-            "count": 3,
+            "name": "quay.io/redhat-services-prod/app-sre-tenant/gitlab-project-exporter-main/gitlab-project-exporter-main",
+            "namespaces": "app-sre-observability-stage",
+            "count": 1,
         },
         {
             "name": "quay.io/redhatproductsecurity/rapidast",
             "namespaces": "app-sre-pipelines",
             "count": 1,
         },
-    ])
+        {
+            "name": "registry.redhat.io/openshift-pipelines/pipelines-entrypoint-rhel8",
+            "namespaces": "app-sre-pipelines",
+            "count": 3,
+        },
+    ]
 
 
-def testfetch_exclude_pattern(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
+def test_fetch_exclude_pattern(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
     images = fetch_pods_images_from_namespaces(
         namespaces=namespaces,
         oc_map=oc_map,
         thread_pool_size=2,
         exclude_pattern="quay.io/redhat|quay.io/app-sre",
     )
-    assert _to_set(images) == _to_set([
+    assert images == [
         {
             "name": "quay.io/prometheus/blackbox-exporter",
             "namespaces": "app-sre-observability-stage",
@@ -168,10 +163,10 @@ def testfetch_exclude_pattern(namespaces: list[NamespaceV1], oc_map: OCMap) -> N
             "namespaces": "app-sre-pipelines",
             "count": 3,
         },
-    ])
+    ]
 
 
-def testfetch_include_pattern(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
+def test_fetch_include_pattern(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
     images = fetch_pods_images_from_namespaces(
         namespaces=namespaces,
         oc_map=oc_map,
@@ -185,3 +180,41 @@ def testfetch_include_pattern(namespaces: list[NamespaceV1], oc_map: OCMap) -> N
             "count": 3,
         },
     ]
+
+
+def test_fetch_exception(
+    namespaces: list[NamespaceV1], mocker: MockerFixture, observability_pods: list[dict]
+) -> None:
+    oc = mocker.patch("reconcile.utils.oc.OCNative", autospec=True)
+    oc.get_items.side_effect = [observability_pods, Exception("generic error")]
+    oc_map = mocker.patch("reconcile.utils.oc_map.OCMap", autospec=True)
+    oc_map.get_cluster.return_value = oc
+
+    images = fetch_pods_images_from_namespaces(
+        namespaces=namespaces,
+        oc_map=oc_map,
+        thread_pool_size=2,
+    )
+
+    assert images == [
+        {
+            "name": "quay.io/app-sre/internal-redhat-ca",
+            "namespaces": "app-sre-observability-stage",
+            "count": 2,
+        },
+        {
+            "name": "quay.io/prometheus/blackbox-exporter",
+            "namespaces": "app-sre-observability-stage",
+            "count": 1,
+        },
+        {
+            "name": "quay.io/redhat-services-prod/app-sre-tenant/gitlab-project-exporter-main/gitlab-project-exporter-main",
+            "namespaces": "app-sre-observability-stage",
+            "count": 1,
+        },
+        {
+            "name": "error",
+            "namespaces": "app-sre-pipelines/generic error",
+            "count": 1,
+        },
+    ]