qontract-reconcile 0.10.1rc1181__py3-none-any.whl → 0.10.1rc1183__py3-none-any.whl

{qontract_reconcile-0.10.1rc1181.dist-info → qontract_reconcile-0.10.1rc1183.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc1181
+Version: 0.10.1rc1183
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc1181.dist-info → qontract_reconcile-0.10.1rc1183.dist-info}/RECORD

@@ -726,7 +726,7 @@ reconcile/utils/sqs_gateway.py,sha256=XNIf3PY4UCPNufP2Ul0UJj3fKlt5larBba-VTT-41F
 reconcile/utils/state.py,sha256=W0_awkLAPX18hNOF_60o73tkPxDUylqbzYNHfl_sDsk,16386
 reconcile/utils/structs.py,sha256=LcbLEg8WxfRqM6nW7NhcWN0YeqF7SQzxOgntmLs1SgY,352
 reconcile/utils/template.py,sha256=wTvRU4AnAV_o042tD4Mwls2dwWMuk7MKnde3MaCjaYg,331
-reconcile/utils/terraform_client.py,sha256=LjX2U2E0Dglt2S_KA5jWQ_dVC8sPn4FEAh0xW_d6JTk,35953
+reconcile/utils/terraform_client.py,sha256=IQV6JsoGH-Fy987stSJ3_A2f2ys_cIj6s9761krePi4,36035
 reconcile/utils/terrascript_aws_client.py,sha256=q6Ydbjle7K5Z3LYpoRJGXnb50ix6aGN6jLvP2vglPz8,283769
 reconcile/utils/three_way_diff_strategy.py,sha256=oQcHXd9LVhirJfoaOBoHUYuZVGfyL2voKr6KVI34zZE,4833
 reconcile/utils/throughput.py,sha256=iP4UWAe2LVhDo69mPPmgo9nQ7RxHD6_GS8MZe-aSiuM,344
@@ -838,10 +838,11 @@ tools/app_interface_metrics_exporter.py,sha256=zkwkxdAUAxjdc-pzx2_oJXG25fo0Fnyd5
 tools/app_interface_reporter.py,sha256=oZPib4HPq0aZ2Zui1QGJGk6qQdfpeihujGDBnSdKyGE,17627
 tools/glitchtip_access_reporter.py,sha256=oPBnk_YoDuljU3v0FaChzOwwnk4vap1xEE67QEjzdqs,2948
 tools/glitchtip_access_revalidation.py,sha256=8kbBJk04mkq28kWoRDDkfCGIF3GRg3pJrFAh1sW0dbk,2821
-tools/qontract_cli.py,sha256=YMOYkmP9WZFMX8wPxoJZ5ddstK3YyXMMx6ReXnCiH0w,140707
+tools/qontract_cli.py,sha256=KfmjfaCuyd8F68oY1hHd2tEf0PY0kLd6t8sWqR81nmc,142498
 tools/sd_app_sre_alert_report.py,sha256=e9vAdyenUz2f5c8-z-5WY0wv-SJ9aePKDH2r4IwB6pc,5063
 tools/template_validation.py,sha256=qpKYaTgk0GOPGa2Ct5_5sKdwIHtCAKIBGzsMPuJU5fw,3371
 tools/cli_commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tools/cli_commands/container_images_report.py,sha256=PCJIzvUqiYmTdn5xJFcxHocCYp6dprrsJ_lkYdl3ET8,4417
 tools/cli_commands/erv2.py,sha256=469qdhyaf7thpPQ4hJSurvmxBqYDJsoI8H4AigQIF7U,20737
 tools/cli_commands/gpg_encrypt.py,sha256=x02JOMn834z89YSNvr5B-oJky7rR1C0begCkPh45eHk,4958
 tools/cli_commands/systems_and_tools.py,sha256=EMHOF1AtUDaoSk0bbjl6oUKYAz4rTZjIBaF-6E6GspM,16816
@@ -876,12 +877,13 @@ tools/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tools/test/conftest.py,sha256=CsDbu4otrxb7X7kXKKGyV3ZEzu3pCkgjCoCGiHNx6zc,2401
 tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvftCWEEf-g1mfXOtgCog-g,1271
 tools/test/test_erv2.py,sha256=EAS7QuJkHisRVO9bMGxm662L5B6i66wF_mT9PAjVzrU,3128
+tools/test/test_get_container_images.py,sha256=L2XzfmYAd6WZ17UXNnr8Z4iwoGcCvQ0vN6gxAZ7gEws,6097
 tools/test/test_qontract_cli.py,sha256=iuzKbQ6ahinvjoQmQLBrG4shey0z-1rB6qCgS8T6dgU,5789
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc1181.dist-info/METADATA,sha256=pDZwlb7DGvqkX4trR3vqj0NqCsBK-LPoCvo46Uufung,2213
-qontract_reconcile-0.10.1rc1181.dist-info/WHEEL,sha256=bFJAMchF8aTQGUgMZzHJyDDMPTO3ToJ7x23SLJa1SVo,92
-qontract_reconcile-0.10.1rc1181.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc1181.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc1181.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc1183.dist-info/METADATA,sha256=awLi7LWrtIKv-9-jSwBD1xyD2dU782uKtYHtKl89r4w,2213
+qontract_reconcile-0.10.1rc1183.dist-info/WHEEL,sha256=bFJAMchF8aTQGUgMZzHJyDDMPTO3ToJ7x23SLJa1SVo,92
+qontract_reconcile-0.10.1rc1183.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc1183.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc1183.dist-info/RECORD,,

reconcile/utils/terraform_client.py

@@ -68,6 +68,10 @@ class TerraformCommandError(CalledProcessError):
     pass
 
 
+class RdsUpgradeValidationError(Exception):
+    pass
+
+
 class DeletionApprovalExpirationValueError(Exception):
     pass
 
@@ -218,7 +222,7 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
         if disable_deletions_detected:
             raise RuntimeError("Terraform plan has disabled deletions detected")
 
-    @retry()
+    @retry(no_retry_exceptions=RdsUpgradeValidationError)
     def terraform_plan(
         self, spec: TerraformSpec, enable_deletion: bool
     ) -> tuple[bool, list[AccountUser], bool]:
@@ -788,21 +792,19 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
                 None,
             )
             if target is None:
-                logging.error(
+                raise RdsUpgradeValidationError(
                     f"Cannot upgrade RDS instance: {resource_name} "
                     f"from {before_version} to {after_version}"
                 )
-                return
             allow_major_version_upgrade = after.get(
                 "allow_major_version_upgrade",
                 False,
             )
             if target["IsMajorVersionUpgrade"] and not allow_major_version_upgrade:
-                logging.error(
+                raise RdsUpgradeValidationError(
                     "allow_major_version_upgrade is not enabled for upgrading RDS instance: "
                     f"{resource_name} to a new major version."
                 )
-                return
 
             blue_green_update = after.get("blue_green_update", [])
             if blue_green_update and blue_green_update[0]["enabled"]:
@@ -859,36 +861,32 @@ class TerraformClient:  # pylint: disable=too-many-public-methods
             return False
 
         if not is_supported(engine, version):
-            logging.error(
+            raise RdsUpgradeValidationError(
                 f"Cannot upgrade RDS instance: {resource_name}. "
                 f"Engine version {version} is not supported for blue/green updates."
             )
-            return
 
         if replica:
-            logging.error(
+            raise RdsUpgradeValidationError(
                 f"Cannot upgrade RDS instance: {resource_name}. "
                 "Blue/green updates are not supported for instances with read replicas."
             )
-            return
 
         if engine == "postgres" and self._aws_api is not None:
             pg_details = self._aws_api.describe_db_parameter_group(
                 account_name, parameter_group, region_name
             )
             if pg_details.get("rds.logical_replication") != "1":
-                logging.error(
+                raise RdsUpgradeValidationError(
                     f"Cannot upgrade RDS instance: {resource_name}. "
                     f"Blue/green updates require logical replication to be enabled in the Parameter group {parameter_group}."
                 )
-                return
 
         if "storage_type" in changed_fields or "allocated_storage" in changed_fields:
-            logging.error(
+            raise RdsUpgradeValidationError(
                 f"Cannot upgrade RDS instance: {resource_name}. "
                 f"Blue/green updates are not supported when 'storage_type' or 'allocated_storage' has changed."
             )
-            return
 
 
 class TerraformPlanFailed(Exception):

tools/cli_commands/container_images_report.py

@@ -0,0 +1,129 @@
+import re
+from collections import defaultdict
+from collections.abc import Sequence
+from typing import Any
+
+from pydantic import BaseModel
+from sretoolbox.utils import threaded
+
+from reconcile.gql_definitions.common.namespaces_minimal import NamespaceV1
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.typed_queries.namespaces_minimal import get_namespaces_minimal
+from reconcile.utils.oc_filters import filter_namespaces_by_cluster_and_namespace
+from reconcile.utils.oc_map import OCMap, init_oc_map_from_namespaces
+from reconcile.utils.secret_reader import create_secret_reader
+
+IMAGE_NAME_REGEX = re.compile(r"^(?P<name>[a-zA-Z0-9][a-zA-Z0-9/_.-]+)(?:@sha256)?:.+$")
+
+
+class NamespaceImages(BaseModel):
+    namespace_name: str
+    image_names: list[str]
+
+
+def get_all_pods_images(
+    cluster_name: Sequence[str] | None = None,
+    namespace_name: Sequence[str] | None = None,
+    thread_pool_size: int = 10,
+    use_jump_host: bool = True,
+    include_pattern: str | None = None,
+    exclude_pattern: str | None = None,
+) -> list[dict[str, Any]]:
+    """Gets all the images in the clusters/namespaces given. Returns a list of dicts
+    with the following keys:
+      * name: image name
+      * namespaces:  a comma separated list of namespaces where the instance is used
+      * count: number of uses of the image
+    """
+    all_namespaces = get_namespaces_minimal()
+    namespaces = filter_namespaces_by_cluster_and_namespace(
+        namespaces=all_namespaces,
+        cluster_names=cluster_name,
+        namespace_names=namespace_name,
+    )
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+    oc_map = init_oc_map_from_namespaces(
+        namespaces=namespaces,
+        integration="qontract-cli-get-namespace_images",
+        secret_reader=secret_reader,
+        use_jump_host=use_jump_host,
+        thread_pool_size=thread_pool_size,
+        init_projects=True,
+    )
+
+    return fetch_pods_images_from_namespaces(
+        namespaces=namespaces,
+        oc_map=oc_map,
+        exclude_pattern=exclude_pattern,
+        include_pattern=include_pattern,
+        thread_pool_size=thread_pool_size,
+    )
+
+
+def fetch_pods_images_from_namespaces(
+    namespaces: list[NamespaceV1],
+    oc_map: OCMap,
+    include_pattern: str | None = None,
+    exclude_pattern: str | None = None,
+    thread_pool_size: int = 10,
+) -> list[dict[str, Any]]:
+    all_namespace_images = threaded.run(
+        func=_get_namespace_images,
+        iterable=namespaces,
+        thread_pool_size=thread_pool_size,
+        oc_map=oc_map,
+    )
+
+    result: defaultdict = defaultdict(_get_all_images_default)
+    for ni in all_namespace_images:
+        for name in ni.image_names:
+            result[name]["namespaces"].add(ni.namespace_name)
+            result[name]["count"] += 1
+
+    exclude_pattern_compiled: re.Pattern | None = None
+    if exclude_pattern:
+        exclude_pattern_compiled = re.compile(exclude_pattern)
+
+    include_pattern_compiled: re.Pattern | None = None
+    if include_pattern:
+        include_pattern_compiled = re.compile(include_pattern)
+
+    result_filtered_flattened: list[dict[str, Any]] = []
+    for name, value in result.items():
+        if include_pattern_compiled and not include_pattern_compiled.match(name):
+            continue
+        if exclude_pattern_compiled and exclude_pattern_compiled.match(name):
+            continue
+
+        result_filtered_flattened.append({
+            "name": name,
+            "namespaces": ",".join(sorted(value["namespaces"])),
+            "count": value["count"],
+        })
+
+    return result_filtered_flattened
+
+
+def _get_all_images_default() -> dict[str, Any]:
+    return {"namespaces": set(), "count": 0}
+
+
+def _get_namespace_images(ns: NamespaceV1, oc_map: OCMap) -> NamespaceImages:
+    image_names = []
+    oc = oc_map.get_cluster(ns.cluster.name)
+    pod_items = oc.get_items("Pod", namespace=ns.name)
+    for pod in pod_items:
+        containers = pod.get("spec", {}).get("containers", [])
+        containers.extend(pod.get("spec", {}).get("initContainers", []))
+
+        for c in containers:
+            if m := IMAGE_NAME_REGEX.match(c["image"]):
+                image_names.append(m.group("name"))
+
+    return NamespaceImages(
+        namespace_name=ns.name,
+        image_names=image_names,
+    )

tools/qontract_cli.py

@@ -63,9 +63,14 @@ from reconcile.checkpoint import report_invalid_metadata
 from reconcile.cli import (
     TERRAFORM_VERSION,
     TERRAFORM_VERSION_REGEX,
+    cluster_name,
     config_file,
+    namespace_name,
     use_jump_host,
 )
+from reconcile.cli import (
+    threaded as thread_pool_size,
+)
 from reconcile.gql_definitions.advanced_upgrade_service.aus_clusters import (
     query as aus_clusters_query,
 )
@@ -136,7 +141,9 @@ from reconcile.utils.oc import (
     OC_Map,
     OCLogMsg,
 )
-from reconcile.utils.oc_map import init_oc_map_from_clusters
+from reconcile.utils.oc_map import (
+    init_oc_map_from_clusters,
+)
 from reconcile.utils.ocm import OCM_PRODUCT_ROSA, OCMMap
 from reconcile.utils.ocm_base_client import init_ocm_base_client
 from reconcile.utils.output import print_output
@@ -4313,5 +4320,68 @@ def migrate(ctx, dry_run: bool, skip_build: bool) -> None:
     rich_print(f"[b red]Please remove the temporary directory ({tempdir}) manually!")
 
 
+@get.command(help="Get all container images in app-interface defined namespaces")
+@cluster_name
+@namespace_name
+@thread_pool_size()
+@use_jump_host()
+@click.option("--exclude-pattern", help="Exclude images that match this pattern")
+@click.option("--include-pattern", help="Only include images that match this pattern")
+@click.pass_context
+def container_images(
+    ctx,
+    cluster_name,
+    namespace_name,
+    thread_pool_size,
+    use_jump_host,
+    exclude_pattern,
+    include_pattern,
+):
+    from tools.cli_commands.container_images_report import get_all_pods_images
+
+    results = get_all_pods_images(
+        cluster_name=cluster_name,
+        namespace_name=namespace_name,
+        thread_pool_size=thread_pool_size,
+        use_jump_host=use_jump_host,
+        exclude_pattern=exclude_pattern,
+        include_pattern=include_pattern,
+    )
+
+    if ctx.obj["options"]["output"] == "md":
+        json_table = {
+            "filter": True,
+            "fields": [
+                {"key": "name", "sortable": True},
+                {"key": "namespaces", "sortable": True},
+                {"key": "count", "sortable": True},
+            ],
+            "items": results,
+        }
+
+        print(
+            f"""
+You can view the source of this Markdown to extract the JSON data.
+
+{len(results)} container images found.
+
+exclude-pattern = {exclude_pattern}
+include-pattern = {include_pattern}
+
+```json:table
+{json.dumps(json_table)}
+```
+            """
+        )
+    else:
+        columns = [
+            "name",
+            "namespaces",
+            "count",
+        ]
+        ctx.obj["options"]["sort"] = False
+        print_output(ctx.obj["options"], results, columns)
+
+
 if __name__ == "__main__":
     root()  # pylint: disable=no-value-for-parameter

tools/test/test_get_container_images.py

@@ -0,0 +1,187 @@
+import pytest
+from pytest_mock import MockerFixture
+
+from reconcile.gql_definitions.common.namespaces_minimal import ClusterV1, NamespaceV1
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+from reconcile.test.fixtures import Fixtures
+from reconcile.utils.oc import OCNative
+from reconcile.utils.oc_map import OCMap
+from tools.cli_commands.container_images_report import (
+    fetch_pods_images_from_namespaces,
+)
+
+fxt = Fixtures("container_images_report")
+
+
+@pytest.fixture
+def observability_pods() -> list[dict]:
+    return fxt.get_anymarkup("app-sre-observability-stage-pods.yaml")
+
+
+@pytest.fixture
+def pipeline_pods() -> list[dict]:
+    return fxt.get_anymarkup("app-sre-pipelines-pods.yaml")
+
+
+@pytest.fixture
+def namespaces() -> list[NamespaceV1]:
+    return [
+        NamespaceV1(
+            name="app-sre-observability-stage",
+            delete=None,
+            labels="{}",
+            clusterAdmin=None,
+            cluster=ClusterV1(
+                name="appsres09ue1",
+                serverUrl="https://api.appsres09ue1.24ep.p3.openshiftapps.com:443",
+                insecureSkipTLSVerify=None,
+                jumpHost=None,
+                automationToken=VaultSecret(
+                    path="app-sre/integrations-output/openshift-cluster-bots/appsres09ue1",
+                    field="token",
+                    version=None,
+                    format=None,
+                ),
+                clusterAdminAutomationToken=VaultSecret(
+                    path="app-sre/integrations-output/openshift-cluster-bots/appsres09ue1-cluster-admin",
+                    field="token",
+                    version=None,
+                    format=None,
+                ),
+                internal=True,
+                disable=None,
+            ),
+        ),
+        NamespaceV1(
+            name="app-sre-pipelines",
+            delete=None,
+            labels='{"provider": "tekton"}',
+            clusterAdmin=None,
+            cluster=ClusterV1(
+                name="appsres09ue1",
+                serverUrl="https://api.appsres09ue1.24ep.p3.openshiftapps.com:443",
+                insecureSkipTLSVerify=None,
+                jumpHost=None,
+                automationToken=VaultSecret(
+                    path="app-sre/integrations-output/openshift-cluster-bots/appsres09ue1",
+                    field="token",
+                    version=None,
+                    format=None,
+                ),
+                clusterAdminAutomationToken=VaultSecret(
+                    path="app-sre/integrations-output/openshift-cluster-bots/appsres09ue1-cluster-admin",
+                    field="token",
+                    version=None,
+                    format=None,
+                ),
+                internal=True,
+                disable=None,
+            ),
+        ),
+    ]
+
+
+@pytest.fixture
+def oc(
+    mocker: MockerFixture,
+    observability_pods: list[dict],
+    pipeline_pods: list[dict],
+) -> OCNative:
+    oc = mocker.patch("reconcile.utils.oc.OCNative", autospec=True)
+    oc.get_items.side_effect = [observability_pods, pipeline_pods]
+    return oc
+
+
+@pytest.fixture
+def oc_map(mocker: MockerFixture, oc: OCNative) -> OCMap:
+    oc_map = mocker.patch("reconcile.utils.oc_map.OCMap", autospec=True)
+    oc_map.get_cluster.return_value = oc
+    return oc_map
+
+
+# convert a list of dicts into a set of tuples to use it in assertions
+def _to_set(list_of_dicts: list[dict]) -> set[tuple]:
+    return {tuple(d.items()) for d in list_of_dicts}
+
+
+def testfetch_no_filter(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
+    images = fetch_pods_images_from_namespaces(
+        namespaces=namespaces,
+        oc_map=oc_map,
+        thread_pool_size=2,
+    )
+
+    assert _to_set(images) == _to_set([
+        {
+            "name": "quay.io/prometheus/blackbox-exporter",
+            "namespaces": "app-sre-observability-stage",
+            "count": 1,
+        },
+        {
+            "name": "quay.io/redhat-services-prod/app-sre-tenant/gitlab-project-exporter-main/gitlab-project-exporter-main",
+            "namespaces": "app-sre-observability-stage",
+            "count": 1,
+        },
+        {
+            "name": "quay.io/app-sre/internal-redhat-ca",
+            "namespaces": "app-sre-observability-stage,app-sre-pipelines",
+            "count": 3,
+        },
+        {
+            "name": "quay.io/app-sre/clamav",
+            "namespaces": "app-sre-pipelines",
+            "count": 1,
+        },
+        {
+            "name": "quay.io/redhat-appstudio/clamav-db",
+            "namespaces": "app-sre-pipelines",
+            "count": 1,
+        },
+        {
+            "name": "registry.redhat.io/openshift-pipelines/pipelines-entrypoint-rhel8",
+            "namespaces": "app-sre-pipelines",
+            "count": 3,
+        },
+        {
+            "name": "quay.io/redhatproductsecurity/rapidast",
+            "namespaces": "app-sre-pipelines",
+            "count": 1,
+        },
+    ])
+
+
+def testfetch_exclude_pattern(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
+    images = fetch_pods_images_from_namespaces(
+        namespaces=namespaces,
+        oc_map=oc_map,
+        thread_pool_size=2,
+        exclude_pattern="quay.io/redhat|quay.io/app-sre",
+    )
+    assert _to_set(images) == _to_set([
+        {
+            "name": "quay.io/prometheus/blackbox-exporter",
+            "namespaces": "app-sre-observability-stage",
+            "count": 1,
+        },
+        {
+            "name": "registry.redhat.io/openshift-pipelines/pipelines-entrypoint-rhel8",
+            "namespaces": "app-sre-pipelines",
+            "count": 3,
+        },
+    ])
+
+
+def testfetch_include_pattern(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
+    images = fetch_pods_images_from_namespaces(
+        namespaces=namespaces,
+        oc_map=oc_map,
+        thread_pool_size=2,
+        include_pattern="^registry.redhat.io",
+    )
+    assert images == [
+        {
+            "name": "registry.redhat.io/openshift-pipelines/pipelines-entrypoint-rhel8",
+            "namespaces": "app-sre-pipelines",
+            "count": 3,
+        },
+    ]