qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

tools/app_interface_reporter.py

@@ -1,9 +1,10 @@
+import contextlib
 import logging
 import os
 import textwrap
 from datetime import (
+    UTC,
     datetime,
-    timezone,
 )
 
 import click
@@ -13,8 +14,8 @@ from dateutil.relativedelta import relativedelta
 from prometheus_client.parser import text_string_to_metric_families
 from sretoolbox.utils.threaded import run
 
-import reconcile.jenkins_plugins as jenkins_base
 from reconcile import (
+    jenkins_base,
     mr_client_gateway,
     queries,
 )
@@ -25,14 +26,7 @@ from reconcile.cli import (
     log_level,
     threaded,
 )
-from reconcile.jenkins_job_builder import (
-    get_openshift_saas_deploy_job_name,
-    init_jjb,
-)
-from reconcile.typed_queries.saas_files import (
-    export_model,
-    get_saas_files,
-)
+from reconcile.jenkins_job_builder import init_jjb
 from reconcile.utils.jjb_client import JJB
 from reconcile.utils.mr import CreateAppInterfaceReporter
 from reconcile.utils.runtime.environment import init_env
@@ -148,13 +142,11 @@ class Report:
 
         for cluster, namespaces in container_vulnerabilities.items():
             for namespace, severities in namespaces.items():
-                parsed_metrics.append(
-                    {
-                        "cluster": cluster,
-                        "namespace": namespace,
-                        "vulnerabilities": severities,
-                    }
-                )
+                parsed_metrics.append({
+                    "cluster": cluster,
+                    "namespace": namespace,
+                    "vulnerabilities": severities,
+                })
         return parsed_metrics
 
     @staticmethod
@@ -165,13 +157,11 @@ class Report:
 
         for cluster, namespaces in post_deploy_jobs.items():
             for namespace, post_deploy_job in namespaces.items():
-                results.append(
-                    {
-                        "cluster": cluster,
-                        "namespace": namespace,
-                        "post_deploy_job": post_deploy_job,
-                    }
-                )
+                results.append({
+                    "cluster": cluster,
+                    "namespace": namespace,
+                    "post_deploy_job": post_deploy_job,
+                })
         return results
 
     @staticmethod
@@ -182,13 +172,11 @@ class Report:
 
         for cluster, namespaces in deployment_validations.items():
             for namespace, validations in namespaces.items():
-                parsed_metrics.append(
-                    {
-                        "cluster": cluster,
-                        "namespace": namespace,
-                        "validations": validations,
-                    }
-                )
+                parsed_metrics.append({
+                    "cluster": cluster,
+                    "namespace": namespace,
+                    "validations": validations,
+                })
         return parsed_metrics
 
     @staticmethod
@@ -231,11 +219,10 @@ def get_apps_data(date, month_delta=1, thread_pool_size=10):
     secret_reader = SecretReader(settings)
 
     apps = queries.get_apps()
-    saas_files = [export_model(saas_file) for saas_file in get_saas_files()]
     jjb: JJB = init_jjb(secret_reader)
     jenkins_map = jenkins_base.get_jenkins_map()
     time_limit = date - relativedelta(months=month_delta)
-    timestamp_limit = int(time_limit.replace(tzinfo=timezone.utc).timestamp())
+    timestamp_limit = int(time_limit.replace(tzinfo=UTC).timestamp())
 
     secret_content = secret_reader.read_all({"path": DASHDOTDB_SECRET})
     dashdotdb_url = secret_content["url"]
@@ -256,22 +243,6 @@ def get_apps_data(date, month_delta=1, thread_pool_size=10):
     build_jobs = jjb.get_all_jobs(job_types=["build"])
     jobs_to_get = build_jobs.copy()
 
-    saas_deploy_jobs = []
-    for saas_file in saas_files:
-        saas_file_name = saas_file["name"]
-        for template in saas_file["resourceTemplates"]:
-            for target in template["targets"]:
-                job = {}
-                job["env"] = target["namespace"]["environment"]["name"]
-                job["app"] = target["namespace"]["app"]["name"]
-                job["cluster"] = target["namespace"]["cluster"]["name"]
-                job["namespace"] = target["namespace"]["name"]
-                job["name"] = get_openshift_saas_deploy_job_name(
-                    saas_file_name, job["env"], settings
-                )
-                job["saas_file_name"] = saas_file_name
-                saas_deploy_jobs.append(job)
-
     job_history = get_build_history_pool(
         jenkins_map, jobs_to_get, timestamp_limit, thread_pool_size
     )
@@ -283,84 +254,16 @@ def get_apps_data(date, month_delta=1, thread_pool_size=10):
         app_name = app["name"]
 
         logging.info(f"collecting post-deploy jobs " f"information for {app_name}")
-        post_deploy_jobs = {}
-        for saas_file in saas_files:
-            if saas_file["app"]["name"] != app_name:
-                continue
-            resource_types = saas_file["managedResourceTypes"]
-
-            # Only jobs of these types are expected to have a
-            # further post-deploy job
-            if not any(
-                [
-                    "Deployment" in resource_types,
-                    "DeploymentConfig" not in resource_types,
-                ]
-            ):
-                continue
-
-            for resource_template in saas_file["resourceTemplates"]:
-                for target in resource_template["targets"]:
-                    cluster = target["namespace"]["cluster"]["name"]
-                    namespace = target["namespace"]["name"]
-                    post_deploy_jobs[cluster] = {}
-                    post_deploy_jobs[cluster][namespace] = False
-
-        for saas_file in saas_files:
-            if saas_file["app"]["name"] != app_name:
-                continue
-            resource_types = saas_file["managedResourceTypes"]
-            if "Job" not in resource_types:
-                continue
-            for resource_template in saas_file["resourceTemplates"]:
-                for target in resource_template["targets"]:
-
-                    cluster = target["namespace"]["cluster"]["name"]
-                    namespace = target["namespace"]["name"]
-
-                    # This block skips the check if the cluster/namespace
-                    # has no Deployment/DeploymentConfig job associated.
-                    if cluster not in post_deploy_jobs:
-                        continue
-                    if namespace not in post_deploy_jobs[cluster]:
-                        continue
-
-                    # Post-deploy job must depend on a openshift-saas-deploy
-                    # job
-                    if target["upstream"] is None or target["upstream"]["name"] is None:
-                        continue
-                    if target["upstream"]["name"].startswith("openshift-saas-deploy-"):
-                        post_deploy_jobs[cluster][namespace] = True
-
-        app["post_deploy_jobs"] = post_deploy_jobs
+        # this is now empty as it referred to post_deploy jobs via Jenkins. This section
+        # should be removed when we publish a new content format or if we get promotion data
+        # differently.
+        app["post_deploy_jobs"] = {}
 
         logging.info(f"collecting promotion history for {app_name}")
         app["promotions"] = {}
-        for job in saas_deploy_jobs:
-            if job["app"] != app_name:
-                continue
-            if job["name"] not in job_history:
-                continue
-            history = job_history[job["name"]]
-            saas_file_name = job["saas_file_name"]
-            if saas_file_name not in app["promotions"]:
-                app["promotions"][saas_file_name] = [
-                    {
-                        "env": job["env"],
-                        "cluster": job["cluster"],
-                        "namespace": job["namespace"],
-                        **history,
-                    }
-                ]
-            else:
-                app["promotions"][saas_file_name].append(
-                    {
-                        "env": job["env"],
-                        "cluster": job["cluster"],
-                        "namespace": job["namespace"],
-                        **history,
-                    }
-                )
+        # this is now empty as it referred to saas files promotions via Jenkins. This section
+        # should be removed when we publish a new content format or if we get promotion data
+        # differently.
 
         logging.info(f"collecting merge activity for {app_name}")
         app["merge_activity"] = {}
@@ -383,9 +286,10 @@ def get_apps_data(date, month_delta=1, thread_pool_size=10):
                         {"branch": job["branch"], **history}
                     ]
                 else:
-                    app["merge_activity"][repo_url].append(
-                        {"branch": job["branch"], **history}
-                    )
+                    app["merge_activity"][repo_url].append({
+                        "branch": job["branch"],
+                        **history,
+                    })
 
         logging.info(f"collecting dashdotdb information for {app_name}")
         app_namespaces = []
@@ -434,16 +338,10 @@ def get_apps_data(date, month_delta=1, thread_pool_size=10):
                             validt_mx[cluster][namespace] = {}
                         if validation not in validt_mx[cluster][namespace]:
                             validt_mx[cluster][namespace][validation] = {}
-                        if (
-                            status not in validt_mx[cluster][namespace][validation]
-                        ):  # noqa: E501
-                            validt_mx[cluster][namespace][validation][
-                                status
-                            ] = {}  # noqa: E501
+                        if status not in validt_mx[cluster][namespace][validation]:
+                            validt_mx[cluster][namespace][validation][status] = {}
                         value = int(sample.value)
-                        validt_mx[cluster][namespace][validation][
-                            status
-                        ] = value  # noqa: E501
+                        validt_mx[cluster][namespace][validation][status] = value
         for family in text_string_to_metric_families(slo_metrics):
             for sample in family.samples:
                 if sample.name == "serviceslometrics":
@@ -460,24 +358,16 @@ def get_apps_data(date, month_delta=1, thread_pool_size=10):
                             slo_mx[cluster] = {}
                         if namespace not in slo_mx[cluster]:
                             slo_mx[cluster][namespace] = {}
-                        if (
-                            slo_doc_name not in slo_mx[cluster][namespace]
-                        ):  # pylint: disable=line-too-long # noqa: E501
+                        if slo_doc_name not in slo_mx[cluster][namespace]:  # pylint: disable=line-too-long
                             slo_mx[cluster][namespace][slo_doc_name] = {}
-                        if (
-                            slo_name not in slo_mx[cluster][namespace][slo_doc_name]
-                        ):  # noqa: E501
-                            slo_mx[cluster][namespace][slo_doc_name][
-                                slo_name
-                            ] = {  # noqa: E501
+                        if slo_name not in slo_mx[cluster][namespace][slo_doc_name]:
+                            slo_mx[cluster][namespace][slo_doc_name][slo_name] = {
                                 sample.labels["type"]: sample.value
                             }
                         else:
-                            slo_mx[cluster][namespace][slo_doc_name][slo_name].update(
-                                {  # pylint: disable=line-too-long # noqa: E501
-                                    sample.labels["type"]: sample.value
-                                }
-                            )
+                            slo_mx[cluster][namespace][slo_doc_name][slo_name].update({  # pylint: disable=line-too-long
+                                sample.labels["type"]: sample.value
+                            })
         app["container_vulnerabilities"] = vuln_mx
         app["deployment_validations"] = validt_mx
         app["service_slo"] = slo_mx
@@ -498,9 +388,9 @@ def get_build_history(job):
 
 def get_build_history_pool(jenkins_map, jobs, timestamp_limit, thread_pool_size):
     history_to_get = []
-    for instance, jobs in jobs.items():
+    for instance, _jobs in jobs.items():
         jenkins = jenkins_map[instance]
-        for job in jobs:
+        for job in _jobs:
             job["jenkins"] = jenkins
             job["timestamp_limit"] = timestamp_limit
             history_to_get.append(job)
@@ -536,7 +426,6 @@ def get_repo_url(job):
 def main(
     configfile, dry_run, log_level, gitlab_project_id, reports_path, thread_pool_size
 ):
-
     init_env(log_level=log_level, config_file=configfile)
 
     now = datetime.now()
@@ -550,12 +439,10 @@ def main(
         if reports_path:
             report_file = os.path.join(reports_path, report["file_path"])
 
-            try:
+            with contextlib.suppress(FileExistsError):
                 os.makedirs(os.path.dirname(report_file))
-            except FileExistsError:
-                pass
 
-            with open(report_file, "w") as f:
+            with open(report_file, "w", encoding="locale") as f:
                 f.write(report["content"])
 
     if not dry_run:

tools/cli_commands/container_images_report.py

@@ -0,0 +1,154 @@
+import re
+from collections import defaultdict
+from collections.abc import Sequence
+from typing import Any
+
+from pydantic import BaseModel
+from sretoolbox.utils import threaded
+
+from reconcile.gql_definitions.common.namespaces_minimal import NamespaceV1
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.typed_queries.namespaces_minimal import get_namespaces_minimal
+from reconcile.utils.oc_filters import filter_namespaces_by_cluster_and_namespace
+from reconcile.utils.oc_map import OCMap, init_oc_map_from_namespaces
+from reconcile.utils.secret_reader import create_secret_reader
+
+IMAGE_NAME_REGEX = re.compile(
+    r"^(?P<name>[a-zA-Z0-9][a-zA-Z0-9/_.-]+)(?:$|(?:@sha256)?:.+.$)"
+)
+
+
+class NamespaceImages(BaseModel):
+    namespace_name: str
+    image_names: list[str] | None = None
+    error_message: str | None = None
+
+
+def get_all_pods_images(
+    cluster_name: Sequence[str] | None = None,
+    namespace_name: Sequence[str] | None = None,
+    thread_pool_size: int = 10,
+    use_jump_host: bool = True,
+    include_pattern: str | None = None,
+    exclude_pattern: str | None = None,
+) -> list[dict[str, Any]]:
+    """Gets all the images in the clusters/namespaces given. Returns a list of dicts
+    with the following keys:
+      * name: image name
+      * namespaces:  a comma separated list of namespaces where the instance is used
+      * count: number of uses of the image
+    """
+    all_namespaces = get_namespaces_minimal()
+    namespaces = filter_namespaces_by_cluster_and_namespace(
+        namespaces=all_namespaces,
+        cluster_names=cluster_name,
+        namespace_names=namespace_name,
+    )
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+    oc_map = init_oc_map_from_namespaces(
+        namespaces=namespaces,
+        integration="qontract-cli-get-namespace_images",
+        secret_reader=secret_reader,
+        use_jump_host=use_jump_host,
+        thread_pool_size=thread_pool_size,
+        init_projects=True,
+    )
+
+    return fetch_pods_images_from_namespaces(
+        namespaces=namespaces,
+        oc_map=oc_map,
+        exclude_pattern=exclude_pattern,
+        include_pattern=include_pattern,
+        thread_pool_size=thread_pool_size,
+    )
+
+
+def fetch_pods_images_from_namespaces(
+    namespaces: list[NamespaceV1],
+    oc_map: OCMap,
+    include_pattern: str | None = None,
+    exclude_pattern: str | None = None,
+    thread_pool_size: int = 10,
+) -> list[dict[str, Any]]:
+    all_namespace_images = threaded.run(
+        func=_get_namespace_images,
+        iterable=namespaces,
+        thread_pool_size=thread_pool_size,
+        oc_map=oc_map,
+    )
+
+    errors: defaultdict = defaultdict(int)
+    result: defaultdict = defaultdict(_get_all_images_default)
+    for ni in all_namespace_images:
+        if ni.error_message:
+            errors[f"{ni.namespace_name}/{ni.error_message}"] += 1
+            continue
+
+        for name in ni.image_names or []:
+            result[name]["namespaces"].add(ni.namespace_name)
+            result[name]["count"] += 1
+
+    exclude_pattern_compiled: re.Pattern | None = None
+    if exclude_pattern:
+        exclude_pattern_compiled = re.compile(exclude_pattern)
+
+    include_pattern_compiled: re.Pattern | None = None
+    if include_pattern:
+        include_pattern_compiled = re.compile(include_pattern)
+
+    result_filtered_flattened: list[dict[str, Any]] = []
+    for name, value in sorted(result.items()):
+        if include_pattern_compiled and not include_pattern_compiled.match(name):
+            continue
+        if exclude_pattern_compiled and exclude_pattern_compiled.match(name):
+            continue
+
+        result_filtered_flattened.append({
+            "name": name,
+            "namespaces": ",".join(sorted(value["namespaces"])),
+            "count": value["count"],
+        })
+
+    # append errors if they exist in the filtered result
+    # not very canonical, but it is better than ignoring them
+    if errors:
+        for message, count in sorted(errors.items()):
+            result_filtered_flattened.append({
+                "name": "error",
+                "namespaces": message,
+                "count": count,
+            })
+
+    return result_filtered_flattened
+
+
+def _get_all_images_default() -> dict[str, Any]:
+    return {"namespaces": set(), "count": 0}
+
+
+def _get_namespace_images(ns: NamespaceV1, oc_map: OCMap) -> NamespaceImages:
+    image_names = []
+
+    try:
+        oc = oc_map.get_cluster(ns.cluster.name)
+        pod_items = oc.get_items("Pod", namespace=ns.name)
+        for pod in pod_items:
+            containers = pod.get("spec", {}).get("containers", [])
+            containers.extend(pod.get("spec", {}).get("initContainers", []))
+
+            for c in containers:
+                if m := IMAGE_NAME_REGEX.match(c["image"]):
+                    image_names.append(m.group("name"))
+    except Exception as exc:
+        return NamespaceImages(
+            namespace_name=ns.name,
+            error_message=str(exc),
+        )
+
+    return NamespaceImages(
+        namespace_name=ns.name,
+        image_names=image_names,
+    )

tools/cli_commands/cost_report/aws.py

@@ -0,0 +1,137 @@
+from collections.abc import Iterable, Mapping
+from decimal import Decimal
+from typing import Self
+
+from sretoolbox.utils import threaded
+
+from reconcile.typed_queries.cost_report.app_names import App, get_app_names
+from reconcile.utils import gql
+from tools.cli_commands.cost_report.cost_management_api import CostManagementApi
+from tools.cli_commands.cost_report.model import ChildAppReport, Report, ReportItem
+from tools.cli_commands.cost_report.response import AwsReportCostResponse
+from tools.cli_commands.cost_report.util import (
+    fetch_cost_report_secret,
+    process_reports,
+)
+from tools.cli_commands.cost_report.view import render_aws_cost_report
+
+THREAD_POOL_SIZE = 10
+
+
+class AwsCostReportCommand:
+    def __init__(
+        self,
+        gql_api: gql.GqlApi,
+        cost_management_api: CostManagementApi,
+        cost_management_console_base_url: str,
+        thread_pool_size: int = THREAD_POOL_SIZE,
+    ) -> None:
+        self.gql_api = gql_api
+        self.cost_management_api = cost_management_api
+        self.cost_management_console_base_url = cost_management_console_base_url
+        self.thread_pool_size = thread_pool_size
+
+    def execute(self) -> str:
+        apps = self.get_apps()
+        responses = self.get_reports(apps)
+        reports = self.process_reports(apps, responses)
+        return self.render(reports)
+
+    def get_apps(self) -> list[App]:
+        """
+        Get all apps from the gql API.
+        """
+        return get_app_names(self.gql_api)
+
+    def _get_report(self, app: App) -> tuple[str, AwsReportCostResponse]:
+        return app.name, self.cost_management_api.get_aws_costs_report(app.name)
+
+    def get_reports(
+        self,
+        apps: Iterable[App],
+    ) -> Mapping[str, AwsReportCostResponse]:
+        """
+        Fetch reports from cost management API
+        """
+        results = threaded.run(self._get_report, apps, self.thread_pool_size)
+        return dict(results)
+
+    def process_reports(
+        self,
+        apps: Iterable[App],
+        responses: Mapping[str, AwsReportCostResponse],
+    ) -> dict[str, Report]:
+        """
+        Build reports with parent-child app tree.
+        """
+        return process_reports(
+            apps,
+            responses,
+            report_builder=self._build_report,
+        )
+
+    def render(self, reports: Mapping[str, Report]) -> str:
+        return render_aws_cost_report(
+            reports=reports,
+            cost_management_console_base_url=self.cost_management_console_base_url,
+        )
+
+    @staticmethod
+    def _build_report(
+        app_name: str,
+        parent_app_name: str | None,
+        child_apps: list[str],
+        reports: Mapping[str, Report],
+        response: AwsReportCostResponse,
+    ) -> Report:
+        child_app_reports = [
+            ChildAppReport(
+                name=child_app,
+                total=reports[child_app].total,
+            )
+            for child_app in child_apps
+        ]
+        child_apps_total = Decimal(
+            sum(child_app.total for child_app in child_app_reports)
+        )
+
+        items = [
+            ReportItem(
+                name=service.service,
+                delta_value=value.delta_value,
+                delta_percent=value.delta_percent,
+                total=value.cost.total.value,
+            )
+            for data in response.data
+            for service in data.services
+            if len(service.values) == 1 and (value := service.values[0]) is not None
+        ]
+
+        items_total = response.meta.total.cost.total.value
+        total = items_total + child_apps_total
+        date = next((d for data in response.data if (d := data.date)), "")
+
+        return Report(
+            app_name=app_name,
+            child_apps=child_app_reports,
+            child_apps_total=child_apps_total,
+            date=date,
+            parent_app_name=parent_app_name,
+            items_delta_value=response.meta.delta.value,
+            items_delta_percent=response.meta.delta.percent,
+            items_total=items_total,
+            total=total,
+            items=items,
+        )
+
+    @classmethod
+    def create(cls) -> Self:
+        gql_api = gql.get_api()
+        secret = fetch_cost_report_secret(gql_api)
+        cost_management_api = CostManagementApi.create_from_secret(secret)
+        return cls(
+            gql_api=gql_api,
+            cost_management_api=cost_management_api,
+            cost_management_console_base_url=secret["console_base_url"],
+            thread_pool_size=THREAD_POOL_SIZE,
+        )