qontract-reconcile 0.10.1rc1184__py3-none-any.whl → 0.10.1rc1186__py3-none-any.whl

{qontract_reconcile-0.10.1rc1184.dist-info → qontract_reconcile-0.10.1rc1186.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc1184
+Version: 0.10.1rc1186
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc1184.dist-info → qontract_reconcile-0.10.1rc1186.dist-info}/RECORD

@@ -655,7 +655,7 @@ reconcile/unleash_feature_toggles/integration.py,sha256=nx7BhtzCsTfPbOp60vI5MkNw
 reconcile/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/aggregated_list.py,sha256=km0xadW0jO4G_CqZPsXmoBURQ8c90FaTu5x4X1K1cZs,3357
 reconcile/utils/amtool.py,sha256=ngtBuVPETH6oAy5RnKzvreVbjwQCaATS_PYYwBprzjQ,2288
-reconcile/utils/aws_api.py,sha256=Mp5-euZUfKfnVzgMZd3LoWbegm1OrNjzpP1A-n2EiF0,67640
+reconcile/utils/aws_api.py,sha256=8LeEweWeydLJB9t-neYkYSN6EneDJontGwcglg0xmS0,67652
 reconcile/utils/aws_helper.py,sha256=MDbv5jrNdqqJ5pfBxniGdJXBBO_EYc2_Uf2w9ZzeMNs,2854
 reconcile/utils/batches.py,sha256=TtEm64a8lWhFuNbUVpFEmXVdU2Q0sTBrP_I0Cjbgh7g,320
 reconcile/utils/binary.py,sha256=7MaAFBpzuBUTJ_aA6G6-eult_BPMVyiXbBLD0Y6F-DM,2301
@@ -842,7 +842,7 @@ tools/qontract_cli.py,sha256=KfmjfaCuyd8F68oY1hHd2tEf0PY0kLd6t8sWqR81nmc,142498
 tools/sd_app_sre_alert_report.py,sha256=e9vAdyenUz2f5c8-z-5WY0wv-SJ9aePKDH2r4IwB6pc,5063
 tools/template_validation.py,sha256=qpKYaTgk0GOPGa2Ct5_5sKdwIHtCAKIBGzsMPuJU5fw,3371
 tools/cli_commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tools/cli_commands/container_images_report.py,sha256=PCJIzvUqiYmTdn5xJFcxHocCYp6dprrsJ_lkYdl3ET8,4417
+tools/cli_commands/container_images_report.py,sha256=64LXkv3eoWtMFZwBgkYE9jSnReDD7A9M0GEVXsR9aGk,5183
 tools/cli_commands/erv2.py,sha256=469qdhyaf7thpPQ4hJSurvmxBqYDJsoI8H4AigQIF7U,20737
 tools/cli_commands/gpg_encrypt.py,sha256=x02JOMn834z89YSNvr5B-oJky7rR1C0begCkPh45eHk,4958
 tools/cli_commands/systems_and_tools.py,sha256=EMHOF1AtUDaoSk0bbjl6oUKYAz4rTZjIBaF-6E6GspM,16816
@@ -877,13 +877,13 @@ tools/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tools/test/conftest.py,sha256=CsDbu4otrxb7X7kXKKGyV3ZEzu3pCkgjCoCGiHNx6zc,2401
 tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvftCWEEf-g1mfXOtgCog-g,1271
 tools/test/test_erv2.py,sha256=EAS7QuJkHisRVO9bMGxm662L5B6i66wF_mT9PAjVzrU,3128
-tools/test/test_get_container_images.py,sha256=L2XzfmYAd6WZ17UXNnr8Z4iwoGcCvQ0vN6gxAZ7gEws,6097
+tools/test/test_get_container_images.py,sha256=QUSb0PjZnL35_dhniMux1_D2G_2vvpcasr8tpwPLwIQ,7125
 tools/test/test_qontract_cli.py,sha256=iuzKbQ6ahinvjoQmQLBrG4shey0z-1rB6qCgS8T6dgU,5789
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc1184.dist-info/METADATA,sha256=8DHsuGLxV4LuSR7LPUvgLMjGqffzD8xy5iCyTv2Ltuc,2213
-qontract_reconcile-0.10.1rc1184.dist-info/WHEEL,sha256=bFJAMchF8aTQGUgMZzHJyDDMPTO3ToJ7x23SLJa1SVo,92
-qontract_reconcile-0.10.1rc1184.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc1184.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc1184.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc1186.dist-info/METADATA,sha256=0PW1mUMDbwpICxL8PDkTf4Ne7UZ-R8R0U4Ktr3KAUAQ,2213
+qontract_reconcile-0.10.1rc1186.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+qontract_reconcile-0.10.1rc1186.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc1186.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc1186.dist-info/RECORD,,

{qontract_reconcile-0.10.1rc1184.dist-info → qontract_reconcile-0.10.1rc1186.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.45.0)
+Generator: bdist_wheel (0.45.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

reconcile/utils/aws_api.py

@@ -956,36 +956,43 @@ class AWSApi:  # pylint: disable=too-many-public-methods
                     for s in vpc_subnets
                 ]
             if hcp_vpc_endpoint_sg:
-                endpoints = AWSApi._get_vpc_endpoints(
-                    [
-                        {"Name": "vpc-id", "Values": [vpc_id]},
-                        {
-                            "Name": "tag:AWSEndpointService",
-                            "Values": ["private-router"],
-                        },
-                    ],
-                    assumed_ec2,
+                api_security_group_id = self._get_api_security_group_id(
+                    assumed_ec2, vpc_id
                 )
-                if len(endpoints) > 1:
-                    raise ValueError(
-                        f"exactly one VPC endpoint for private API router in VPC {vpc_id} expected but {len(endpoints)} found"
-                    )
-                vpc_endpoint_id = endpoints[0]["VpcEndpointId"]
-                # https://github.com/openshift/hypershift/blob/c855f68e84e78924ccc9c2132b75dc7e30c4e1d8/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go#L4243
-                security_groups = [
-                    sg
-                    for sg in endpoints[0]["Groups"]
-                    if sg["GroupName"].endswith("-default-sg")
-                ]
-                if len(security_groups) != 1:
-                    raise ValueError(
-                        f"exactly one VPC endpoint default security group for private API router {vpc_endpoint_id} "
-                        f"in VPC {vpc_id} expected but {len(security_groups)} found"
-                    )
-                api_security_group_id = security_groups[0]["GroupId"]
 
         return vpc_id, route_table_ids, subnets_id_az, api_security_group_id
 
+    def _get_api_security_group_id(self, assumed_ec2, vpc_id):
+        endpoints = AWSApi._get_vpc_endpoints(
+            [
+                {"Name": "vpc-id", "Values": [vpc_id]},
+                {
+                    "Name": "tag:AWSEndpointService",
+                    "Values": ["private-router"],
+                },
+            ],
+            assumed_ec2,
+        )
+        if not endpoints:
+            return None
+        if len(endpoints) > 1:
+            raise ValueError(
+                f"exactly one VPC endpoint for private API router in VPC {vpc_id} expected but {len(endpoints)} found"
+            )
+        vpc_endpoint_id = endpoints[0]["VpcEndpointId"]
+        # https://github.com/openshift/hypershift/blob/c855f68e84e78924ccc9c2132b75dc7e30c4e1d8/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go#L4243
+        security_groups = [
+            sg
+            for sg in endpoints[0]["Groups"]
+            if sg["GroupName"].endswith("-default-sg")
+        ]
+        if len(security_groups) != 1:
+            raise ValueError(
+                f"exactly one VPC endpoint default security group for private API router {vpc_endpoint_id} "
+                f"in VPC {vpc_id} expected but {len(security_groups)} found"
+            )
+        return security_groups[0]["GroupId"]
+
     def get_cluster_nat_gateways_egress_ips(self, account: dict[str, Any], vpc_id: str):
         assumed_role_data = self._get_account_assume_data(account)
         assumed_ec2 = self._get_assumed_role_client(*assumed_role_data)

tools/cli_commands/container_images_report.py

@@ -20,7 +20,8 @@ IMAGE_NAME_REGEX = re.compile(r"^(?P<name>[a-zA-Z0-9][a-zA-Z0-9/_.-]+)(?:@sha256
 
 class NamespaceImages(BaseModel):
     namespace_name: str
-    image_names: list[str]
+    image_names: list[str] | None = None
+    error_message: str | None = None
 
 
 def get_all_pods_images(
@@ -77,9 +78,14 @@ def fetch_pods_images_from_namespaces(
         oc_map=oc_map,
     )
 
+    errors: defaultdict = defaultdict(int)
     result: defaultdict = defaultdict(_get_all_images_default)
     for ni in all_namespace_images:
-        for name in ni.image_names:
+        if ni.error_message:
+            errors[f"{ni.namespace_name}/{ni.error_message}"] += 1
+            continue
+
+        for name in ni.image_names or []:
             result[name]["namespaces"].add(ni.namespace_name)
             result[name]["count"] += 1
 
@@ -92,7 +98,7 @@ def fetch_pods_images_from_namespaces(
         include_pattern_compiled = re.compile(include_pattern)
 
     result_filtered_flattened: list[dict[str, Any]] = []
-    for name, value in result.items():
+    for name, value in sorted(result.items()):
         if include_pattern_compiled and not include_pattern_compiled.match(name):
             continue
         if exclude_pattern_compiled and exclude_pattern_compiled.match(name):
@@ -104,6 +110,16 @@ def fetch_pods_images_from_namespaces(
             "count": value["count"],
         })
 
+    # append errors if they exist in the filtered result
+    # not very canonical, but it is better than ignoring them
+    if errors:
+        for message, count in sorted(errors.items()):
+            result_filtered_flattened.append({
+                "name": "error",
+                "namespaces": message,
+                "count": count,
+            })
+
     return result_filtered_flattened
 
 
@@ -113,15 +129,22 @@ def _get_all_images_default() -> dict[str, Any]:
 
 def _get_namespace_images(ns: NamespaceV1, oc_map: OCMap) -> NamespaceImages:
     image_names = []
-    oc = oc_map.get_cluster(ns.cluster.name)
-    pod_items = oc.get_items("Pod", namespace=ns.name)
-    for pod in pod_items:
-        containers = pod.get("spec", {}).get("containers", [])
-        containers.extend(pod.get("spec", {}).get("initContainers", []))
-
-        for c in containers:
-            if m := IMAGE_NAME_REGEX.match(c["image"]):
-                image_names.append(m.group("name"))
+
+    try:
+        oc = oc_map.get_cluster(ns.cluster.name)
+        pod_items = oc.get_items("Pod", namespace=ns.name)
+        for pod in pod_items:
+            containers = pod.get("spec", {}).get("containers", [])
+            containers.extend(pod.get("spec", {}).get("initContainers", []))
+
+            for c in containers:
+                if m := IMAGE_NAME_REGEX.match(c["image"]):
+                    image_names.append(m.group("name"))
+    except Exception as exc:
+        return NamespaceImages(
+            namespace_name=ns.name,
+            error_message=str(exc),
+        )
 
     return NamespaceImages(
         namespace_name=ns.name,

tools/test/test_get_container_images.py

@@ -99,27 +99,17 @@ def oc_map(mocker: MockerFixture, oc: OCNative) -> OCMap:
     return oc_map
 
 
-# convert a list of dicts into a set of tuples to use it in assertions
-def _to_set(list_of_dicts: list[dict]) -> set[tuple]:
-    return {tuple(d.items()) for d in list_of_dicts}
-
-
-def testfetch_no_filter(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
+def test_fetch_no_filter(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
     images = fetch_pods_images_from_namespaces(
         namespaces=namespaces,
         oc_map=oc_map,
         thread_pool_size=2,
     )
 
-    assert _to_set(images) == _to_set([
-        {
-            "name": "quay.io/prometheus/blackbox-exporter",
-            "namespaces": "app-sre-observability-stage",
-            "count": 1,
-        },
+    assert images == [
         {
-            "name": "quay.io/redhat-services-prod/app-sre-tenant/gitlab-project-exporter-main/gitlab-project-exporter-main",
-            "namespaces": "app-sre-observability-stage",
+            "name": "quay.io/app-sre/clamav",
+            "namespaces": "app-sre-pipelines",
             "count": 1,
         },
         {
@@ -128,8 +118,8 @@ def testfetch_no_filter(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
             "count": 3,
         },
         {
-            "name": "quay.io/app-sre/clamav",
-            "namespaces": "app-sre-pipelines",
+            "name": "quay.io/prometheus/blackbox-exporter",
+            "namespaces": "app-sre-observability-stage",
             "count": 1,
         },
         {
@@ -138,26 +128,31 @@ def testfetch_no_filter(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
             "count": 1,
         },
         {
-            "name": "registry.redhat.io/openshift-pipelines/pipelines-entrypoint-rhel8",
-            "namespaces": "app-sre-pipelines",
-            "count": 3,
+            "name": "quay.io/redhat-services-prod/app-sre-tenant/gitlab-project-exporter-main/gitlab-project-exporter-main",
+            "namespaces": "app-sre-observability-stage",
+            "count": 1,
         },
         {
             "name": "quay.io/redhatproductsecurity/rapidast",
             "namespaces": "app-sre-pipelines",
             "count": 1,
         },
-    ])
+        {
+            "name": "registry.redhat.io/openshift-pipelines/pipelines-entrypoint-rhel8",
+            "namespaces": "app-sre-pipelines",
+            "count": 3,
+        },
+    ]
 
 
-def testfetch_exclude_pattern(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
+def test_fetch_exclude_pattern(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
     images = fetch_pods_images_from_namespaces(
         namespaces=namespaces,
         oc_map=oc_map,
         thread_pool_size=2,
         exclude_pattern="quay.io/redhat|quay.io/app-sre",
     )
-    assert _to_set(images) == _to_set([
+    assert images == [
         {
             "name": "quay.io/prometheus/blackbox-exporter",
             "namespaces": "app-sre-observability-stage",
@@ -168,10 +163,10 @@ def testfetch_exclude_pattern(namespaces: list[NamespaceV1], oc_map: OCMap) -> N
             "namespaces": "app-sre-pipelines",
             "count": 3,
         },
-    ])
+    ]
 
 
-def testfetch_include_pattern(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
+def test_fetch_include_pattern(namespaces: list[NamespaceV1], oc_map: OCMap) -> None:
     images = fetch_pods_images_from_namespaces(
         namespaces=namespaces,
         oc_map=oc_map,
@@ -185,3 +180,41 @@ def testfetch_include_pattern(namespaces: list[NamespaceV1], oc_map: OCMap) -> N
             "count": 3,
         },
     ]
+
+
+def test_fetch_exception(
+    namespaces: list[NamespaceV1], mocker: MockerFixture, observability_pods: list[dict]
+) -> None:
+    oc = mocker.patch("reconcile.utils.oc.OCNative", autospec=True)
+    oc.get_items.side_effect = [observability_pods, Exception("generic error")]
+    oc_map = mocker.patch("reconcile.utils.oc_map.OCMap", autospec=True)
+    oc_map.get_cluster.return_value = oc
+
+    images = fetch_pods_images_from_namespaces(
+        namespaces=namespaces,
+        oc_map=oc_map,
+        thread_pool_size=2,
+    )
+
+    assert images == [
+        {
+            "name": "quay.io/app-sre/internal-redhat-ca",
+            "namespaces": "app-sre-observability-stage",
+            "count": 2,
+        },
+        {
+            "name": "quay.io/prometheus/blackbox-exporter",
+            "namespaces": "app-sre-observability-stage",
+            "count": 1,
+        },
+        {
+            "name": "quay.io/redhat-services-prod/app-sre-tenant/gitlab-project-exporter-main/gitlab-project-exporter-main",
+            "namespaces": "app-sre-observability-stage",
+            "count": 1,
+        },
+        {
+            "name": "error",
+            "namespaces": "app-sre-pipelines/generic error",
+            "count": 1,
+        },
+    ]