qontract-reconcile 0.10.2.dev345__py3-none-any.whl → 0.10.2.dev408__py3-none-any.whl

reconcile/utils/oc.py

@@ -12,7 +12,6 @@ import threading
 import time
 from contextlib import suppress
 from dataclasses import dataclass
-from datetime import datetime
 from functools import cache, wraps
 from subprocess import Popen
 from threading import Lock
@@ -47,6 +46,8 @@ from sretoolbox.utils import (
 )
 
 from reconcile.status import RunningState
+from reconcile.utils.datetime_util import utc_now
+from reconcile.utils.json import json_dumps
 from reconcile.utils.jump_host import (
     JumphostParameters,
     JumpHostSSH,
@@ -67,7 +68,8 @@ if TYPE_CHECKING:
 urllib3.disable_warnings()
 
 GET_REPLICASET_MAX_ATTEMPTS = 20
-
+DEFAULT_GROUP = ""
+PROJECT_KIND = "Project.project.openshift.io"
 
 oc_run_execution_counter = Counter(
     name="oc_run_execution_counter",
@@ -144,6 +146,14 @@ class RequestEntityTooLargeError(Exception):
     pass
 
 
+class KindNotFoundError(Exception):
+    pass
+
+
+class AmbiguousResourceTypeError(Exception):
+    pass
+
+
 class OCDecorators:
     @classmethod
     def process_reconcile_time(cls, function: Callable) -> Callable:
@@ -379,10 +389,7 @@ class OCCli:
 
         self.init_projects = init_projects
         if self.init_projects:
-            if self.is_kind_supported("Project"):
-                kind = "Project.project.openshift.io"
-            else:
-                kind = "Namespace"
+            kind = PROJECT_KIND if self.is_kind_supported(PROJECT_KIND) else "Namespace"
             self.projects = {p["metadata"]["name"] for p in self.get_all(kind)["items"]}
 
         self.slow_oc_reconcile_threshold = float(
@@ -452,10 +459,7 @@ class OCCli:
 
         self.init_projects = init_projects
         if self.init_projects:
-            if self.is_kind_supported("Project"):
-                kind = "Project.project.openshift.io"
-            else:
-                kind = "Namespace"
+            kind = PROJECT_KIND if self.is_kind_supported(PROJECT_KIND) else "Namespace"
             self.projects = {p["metadata"]["name"] for p in self.get_all(kind)["items"]}
 
         self.slow_oc_reconcile_threshold = float(
@@ -563,7 +567,7 @@ class OCCli:
             "-f",
             "-",
         ] + parameters_to_process
-        result = self._run(cmd, stdin=json.dumps(template, sort_keys=True))
+        result = self._run(cmd, stdin=json_dumps(template))
         return json.loads(result)["items"]
 
     @OCDecorators.process_reconcile_time
@@ -592,7 +596,7 @@ class OCCli:
     def patch(
         self, namespace: str, kind: str, name: str, patch: Mapping[str, Any]
     ) -> OCProcessReconcileTimeDecoratorMsg:
-        cmd = ["patch", "-n", namespace, kind, name, "-p", json.dumps(patch)]
+        cmd = ["patch", "-n", namespace, kind, name, "-p", json_dumps(patch)]
         self._run(cmd)
         resource = OR({"kind": kind, "metadata": {"name": name}}, "", "")
         return self._msg_to_process_reconcile_time(namespace, resource)
@@ -636,11 +640,9 @@ class OCCli:
     def project_exists(self, name: str) -> bool:
         if name in self.projects:
             return True
+        kind = PROJECT_KIND if self.is_kind_supported(PROJECT_KIND) else "Namespace"
         try:
-            if self.is_kind_supported("Project"):
-                self.get(None, "Project.project.openshift.io", name)
-            else:
-                self.get(None, "Namespace", name)
+            self.get(None, kind, name)
         except StatusCodeError as e:
             if "NotFound" in str(e):
                 return False
@@ -649,7 +651,7 @@ class OCCli:
 
     @OCDecorators.process_reconcile_time
     def new_project(self, namespace: str) -> OCProcessReconcileTimeDecoratorMsg:
-        if self.is_kind_supported("Project"):
+        if self.is_kind_supported(PROJECT_KIND):
             cmd = ["new-project", namespace]
         else:
             cmd = ["create", "namespace", namespace]
@@ -665,7 +667,7 @@ class OCCli:
 
     @OCDecorators.process_reconcile_time
     def delete_project(self, namespace: str) -> OCProcessReconcileTimeDecoratorMsg:
-        if self.is_kind_supported("Project"):
+        if self.is_kind_supported(PROJECT_KIND):
             cmd = ["delete", "project", namespace]
         else:
             cmd = ["delete", "namespace", namespace]
@@ -716,7 +718,7 @@ class OCCli:
         cmd = ["sa", "-n", namespace, "get-token", name]
         return self._run(cmd)
 
-    def get_api_resources(self) -> dict[str, Any]:
+    def get_api_resources(self) -> dict[str, list[OCCliApiResource]]:
         with self.api_resources_lock:
             if not self.api_resources:
                 cmd = ["api-resources", "--no-headers"]
@@ -1009,7 +1011,7 @@ class OCCli:
         name = obj["metadata"]["name"]
         logging.info([f"recycle_{kind.lower()}", self.cluster_name, namespace, name])
         if not dry_run:
-            now = datetime.now()
+            now = utc_now()
             recycle_time = now.strftime("%d/%m/%Y %H:%M:%S")
 
             # get the object in case it was modified
@@ -1020,7 +1022,7 @@ class OCCli:
             a["recycle.time"] = recycle_time
             obj["spec"]["template"]["metadata"]["annotations"] = a
             cmd = ["apply", "-n", namespace, "-f", "-"]
-            stdin = json.dumps(obj, sort_keys=True)
+            stdin = json_dumps(obj)
             self._run(cmd, stdin=stdin, apply=True)
 
     def get_obj_root_owner(
@@ -1195,76 +1197,90 @@ class OCCli:
 
         return out_json
 
-    def _parse_kind(self, kind_name: str) -> tuple[str, str]:
-        # This is a provisional solution while we work in redefining
-        # the api resources initialization.
-        if not self.api_resources:
-            self.get_api_resources()
+    def parse_kind(self, kind: str) -> tuple[str, str, str]:
+        """Parse a Kubernetes kind string into its components.
 
-        kind_group = kind_name.split(".", 1)
-        kind = kind_group[0]
-        if kind in self.api_resources:
-            group_version = self.api_resources[kind][0].group_version
-        else:
-            raise StatusCodeError(f"{self.server}: {kind} does not exist")
-
-        # if a kind_group has more than 1 entry than the kind_name is in
-        # the format kind.apigroup.  Find the apigroup/version that matches
-        # the apigroup passed with the kind_name
-        if len(kind_group) > 1:
-            apigroup_override = kind_group[1]
-            find = False
-            for gv in self.api_resources[kind]:
-                if apigroup_override == gv.group:
-                    if not gv.group:
-                        group_version = gv.api_version
-                    else:
-                        group_version = f"{gv.group}/{gv.api_version}"
-                    find = True
-                    break
+        Supports three formats:
+        - kind
+        - kind.group.whatever
+        - kind.group.whatever/version
 
-            if not find:
-                raise StatusCodeError(
-                    f"{self.server}: {apigroup_override} does not have kind {kind}"
-                )
-        return (kind, group_version)
+        Args:
+            kind: A Kubernetes kind string in one of the supported formats
+
+        Returns:
+            Tuple of (kind, group, version) where missing parts are empty strings
+
+        Raises:
+            ValueError: If the kind string format is invalid
+
+        Examples:
+            >>> parse_kind_string("Deployment")
+            ('Deployment', '', '')
+            >>> parse_kind_string("ClusterRoleBinding.rbac.authorization.k8s.io")
+            ('ClusterRoleBinding', 'rbac.authorization.k8s.io', '')
+            >>> parse_kind_string("CustomResource.mygroup.example.com/v1")
+            ('CustomResource', 'mygroup.example.com', 'v1')
+        """
+        pattern = r"^(?P<kind>[^./]+)(?:\.(?P<group>[^/]+))?(?:/(?P<version>.+))?$"
+        match = re.match(pattern, kind)
+        if not match:
+            raise ValueError(f"Invalid kind string: {kind}")
+
+        kind = match.group("kind") or ""
+        group = match.group("group") or DEFAULT_GROUP
+        version = match.group("version") or ""
+
+        return kind, group, version
 
     def is_kind_supported(self, kind: str) -> bool:
-        # This is a provisional solution while we work in redefining
-        # the api resources initialization.
-        if not self.api_resources:
-            self.get_api_resources()
+        """Returns True if the given kind is supported by the cluster, False otherwise.
 
-        if "." in kind:
-            try:
-                self._parse_kind(kind)
-                return True
-            except StatusCodeError:
-                return False
-        else:
-            return kind in self.api_resources
+        Kind can be either kind, kind.group or kind.group/version."""
+        try:
+            self.get_api_resource(kind)
+            return True
+        except KindNotFoundError:
+            return False
 
     def is_kind_namespaced(self, kind: str) -> bool:
-        # This is a provisional solution while we work in redefining
-        # the api resources initialization.
+        """Returns True if the given kind is namespaced, False if it's cluster scoped.
+
+        Kind can be either kind, kind.group or kind.group/version."""
+        return self.get_api_resource(kind).namespaced
+
+    def get_api_resource(self, kind: str) -> OCCliApiResource:
+        """Return the OCCliApiResource for the given resource type.
+
+        Resource type can be either kind, kind.group or kind.group/version.
+        If kind is not unique, group must be specified."""
+
         if not self.api_resources:
-            self.get_api_resources()
+            raise RuntimeError("API resources not initialized")
 
-        kg = kind.split(".", 1)
-        kind = kg[0]
+        kind, group, _ = self.parse_kind(kind)
 
-        # Same Kinds might exist in different api groups
-        kind_resources = self.api_resources.get(kind)
-        if not kind_resources:
-            raise StatusCodeError(f"Kind {kind} does not exist in the ApiServer")
+        if not (resources := self.api_resources.get(kind)):
+            # the kind not found at all
+            raise KindNotFoundError(f"Unsupported resource type: {kind}")
 
-        if len(kg) > 1:
-            group = kg[1]
-            for r in kind_resources:
-                if group == r.group:
-                    return r.namespaced
-            raise StatusCodeError(f"Kind: {kind} does nod exist in the ApiServer")
-        return kind_resources[0].namespaced
+        if len(resources) == 1 and group == DEFAULT_GROUP:
+            return resources[0]
+
+        # get the resource with the specified group
+        if resource := next((r for r in resources if r.group == group), None):
+            return resource
+
+        # no resource with the specified group found
+        if group == DEFAULT_GROUP:
+            message = (
+                f"Ambiguous resource type: {kind}. "
+                "Please fully qualify it with its API group. E.g., ClusterRoleBinding -> ClusterRoleBinding.rbac.authorization.k8s.io"
+            )
+            raise AmbiguousResourceTypeError(message)
+
+        # group was specified but no matching resource found
+        raise KindNotFoundError(f"Unsupported resource type: {kind}")
 
 
 REQUEST_TIMEOUT = 60
@@ -1304,20 +1320,16 @@ class OCNative(OCCli):
 
             server = connection_parameters.server_url
 
-        if server:
-            self.client = self._get_client(server, token)
-            self.api_resources = self.get_api_resources()
+        if not server:
+            raise Exception("Server name is required!")
 
-        else:
-            raise Exception("A method relies on client/api_kind_version to be set")
+        self.client = self._get_client(server, token)
+        self.api_resources = self.get_api_resources()
 
         self.projects = set()
         self.init_projects = init_projects
         if self.init_projects:
-            if self.is_kind_supported("Project"):
-                kind = "Project.project.openshift.io"
-            else:
-                kind = "Namespace"
+            kind = PROJECT_KIND if self.is_kind_supported(PROJECT_KIND) else "Namespace"
             self.projects = {p["metadata"]["name"] for p in self.get_all(kind)["items"]}
 
     def __enter__(self) -> OCNative:
@@ -1367,8 +1379,10 @@ class OCNative(OCCli):
 
     @retry(max_attempts=5, exceptions=(ServerTimeoutError))
     def get_items(self, kind: str, **kwargs: Any) -> list[dict[str, Any]]:
-        k, group_version = self._parse_kind(kind)
-        obj_client = self._get_obj_client(group_version=group_version, kind=k)
+        resource = self.get_api_resource(kind)
+        obj_client = self._get_obj_client(
+            group_version=resource.group_version, kind=resource.kind
+        )
 
         namespace = ""
         if "namespace" in kwargs:
@@ -1420,8 +1434,10 @@ class OCNative(OCCli):
         name: str | None = None,
         allow_not_found: bool = False,
     ) -> dict[str, Any]:
-        k, group_version = self._parse_kind(kind)
-        obj_client = self._get_obj_client(group_version=group_version, kind=k)
+        resource = self.get_api_resource(kind)
+        obj_client = self._get_obj_client(
+            group_version=resource.group_version, kind=resource.kind
+        )
         try:
             obj = obj_client.get(
                 name=name,
@@ -1435,8 +1451,10 @@ class OCNative(OCCli):
             raise StatusCodeError(f"[{self.server}]: {e}") from None
 
     def get_all(self, kind: str, all_namespaces: bool = False) -> dict[str, Any]:
-        k, group_version = self._parse_kind(kind)
-        obj_client = self._get_obj_client(group_version=group_version, kind=k)
+        resource = self.get_api_resource(kind)
+        obj_client = self._get_obj_client(
+            group_version=resource.group_version, kind=resource.kind
+        )
         try:
             return obj_client.get(_request_timeout=REQUEST_TIMEOUT).to_dict()
         except NotFoundError as e:

reconcile/utils/oc_filters.py

@@ -19,19 +19,19 @@ class Namespace(Protocol):
 NS = TypeVar("NS", bound=Namespace)
 
 
-def filter_namespaces_by_cluster(
+def filter_namespaces_by_cluster[NS: Namespace](
     namespaces: Iterable[NS], cluster_names: Iterable[str]
 ) -> list[NS]:
     return [n for n in namespaces if n.cluster.name in cluster_names]
 
 
-def filter_namespaces_by_name(
+def filter_namespaces_by_name[NS: Namespace](
     namespaces: Iterable[NS], namespace_names: Iterable[str]
 ) -> list[NS]:
     return [n for n in namespaces if n.name in namespace_names]
 
 
-def filter_namespaces_by_cluster_and_namespace(
+def filter_namespaces_by_cluster_and_namespace[NS: Namespace](
     namespaces: Iterable[NS],
     cluster_names: Iterable[str] | None,
     namespace_names: Iterable[str] | None,

reconcile/utils/ocm/products.py

@@ -47,6 +47,7 @@ SPEC_ATTR_MULTI_AZ = "multi_az"
 SPEC_ATTR_HYPERSHIFT = "hypershift"
 SPEC_ATTR_SUBNET_IDS = "subnet_ids"
 SPEC_ATTR_AVAILABILITY_ZONES = "availability_zones"
+SPEC_ATTR_FIPS = "fips"
 
 SPEC_ATTR_NETWORK = "network"
 IGNORE_NETWORK_TYPE_ATTR = "type"
@@ -177,6 +178,7 @@ class OCMProductOsd(OCMProduct):
             ],
             provision_shard_id=provision_shard_id,
             hypershift=cluster["hypershift"]["enabled"],
+            fips=cluster.get("fips") or False,
         )
 
         if not cluster["ccs"]["enabled"]:
@@ -257,6 +259,7 @@ class OCMProductOsd(OCMProduct):
                 if (duwm := cluster.spec.disable_user_workload_monitoring) is not None
                 else True
             ),
+            "fips": cluster.spec.fips,
         }
 
         # Workaround to enable type checks.
@@ -426,6 +429,7 @@ class OCMProductRosa(OCMProduct):
             subnet_ids=cluster["aws"].get("subnet_ids"),
             availability_zones=cluster["nodes"].get("availability_zones"),
             oidc_endpoint_url=oidc_endpoint_url,
+            fips=cluster.get("fips") or False,
         )
 
         machine_pools = [
@@ -513,6 +517,7 @@ class OCMProductRosa(OCMProduct):
                 if (duwm := cluster.spec.disable_user_workload_monitoring) is not None
                 else True
             ),
+            "fips": cluster.spec.fips,
         }
 
         provision_shard_id = cluster.spec.provision_shard_id
@@ -701,6 +706,7 @@ class OCMProductHypershift(OCMProduct):
             availability_zones=cluster["nodes"].get("availability_zones"),
             hypershift=cluster["hypershift"]["enabled"],
             oidc_endpoint_url=oidc_endpoint_url,
+            fips=cluster.get("fips") or False,
         )
 
         network = OCMClusterNetwork(

reconcile/utils/ocm/search_filters.py

@@ -5,7 +5,6 @@ from abc import (
 from collections.abc import Iterable
 from dataclasses import dataclass
 from datetime import (
-    UTC,
     datetime,
 )
 from enum import Enum
@@ -16,6 +15,8 @@ from typing import (
 
 import dateparser
 
+from reconcile.utils.datetime_util import utc_now
+
 
 @dataclass
 class FilterCondition:
@@ -166,17 +167,13 @@ class DateRangeCondition(FilterCondition):
             return date
         parsed = dateparser.parse(
             date,
-            settings={"RELATIVE_BASE": DateRangeCondition.now()},
+            settings={"RELATIVE_BASE": utc_now()},
         )
         if parsed is None:
             raise InvalidFilterError(f"Invalid relative date: {date}")
 
         return parsed
 
-    @staticmethod
-    def now() -> datetime:
-        return datetime.now(tz=UTC)
-
 
 class InvalidFilterError(Exception):
     pass

reconcile/utils/ocm/service_log.py

@@ -1,9 +1,7 @@
 from collections.abc import Generator
-from datetime import (
-    datetime,
-    timedelta,
-)
+from datetime import timedelta
 
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.ocm.base import (
     OCMClusterServiceLog,
     OCMClusterServiceLogCreateModel,
@@ -47,7 +45,7 @@ def create_service_log(
                 .eq("severity", service_log.severity.value)
                 .eq("summary", service_log.summary)
                 .eq("description", service_log.description)
-                .after("created_at", datetime.utcnow() - dedup_interval),
+                .after("created_at", utc_now() - dedup_interval),
             ),
             None,
         )

reconcile/utils/openshift_resource.py

@@ -4,9 +4,8 @@ from __future__ import annotations
 import base64
 import contextlib
 import copy
-import datetime
 import hashlib
-import json
+import logging
 import re
 from threading import Lock
 from typing import TYPE_CHECKING, Any
@@ -15,6 +14,8 @@ import semver
 from pydantic import BaseModel
 
 from reconcile.external_resources.meta import SECRET_UPDATED_AT
+from reconcile.utils.datetime_util import to_utc_seconds_iso_format, utc_now
+from reconcile.utils.json import json_dumps
 from reconcile.utils.metrics import GaugeMetric
 
 if TYPE_CHECKING:
@@ -368,8 +369,8 @@ class OpenshiftResource:
         annotations[QONTRACT_ANNOTATION_INTEGRATION] = self.integration
         annotations[QONTRACT_ANNOTATION_INTEGRATION_VERSION] = self.integration_version
         annotations[QONTRACT_ANNOTATION_SHA256SUM] = sha256sum
-        now = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
-        annotations[QONTRACT_ANNOTATION_UPDATE] = now
+        now = utc_now()
+        annotations[QONTRACT_ANNOTATION_UPDATE] = to_utc_seconds_iso_format(now)
         if self.caller_name:
             annotations[QONTRACT_ANNOTATION_CALLER_NAME] = self.caller_name
 
@@ -530,7 +531,7 @@ class OpenshiftResource:
 
     @staticmethod
     def serialize(body: dict[str, Any]) -> str:
-        return json.dumps(body, sort_keys=True)
+        return json_dumps(body)
 
     @staticmethod
     def calculate_sha256sum(body: str) -> str:
@@ -601,6 +602,10 @@ class ResourceInventory:
         resource: OpenshiftResource,
         privileged: bool = False,
     ) -> None:
+        if cluster not in self._clusters:
+            logging.error(f"Cluster {cluster} not initialized in ResourceInventory")
+            return
+
         if resource.kind_and_group in self._clusters[cluster][namespace]:
             kind = resource.kind_and_group
         else:

reconcile/utils/output.py

@@ -1,10 +1,11 @@
-import json
 import re
 from collections.abc import Iterable, Mapping
 
 import yaml
 from tabulate import tabulate
 
+from reconcile.utils.json import json_dumps
+
 
 def print_output(
     options: Mapping[str, str | bool],
@@ -30,7 +31,7 @@ def print_output(
         )
         print(formatted_content)
     elif output == "json":
-        formatted_content = json.dumps(content)
+        formatted_content = json_dumps(content)
         print(formatted_content)
     elif output == "yaml":
         formatted_content = yaml.dump(content)

reconcile/utils/pagerduty_api.py

@@ -3,8 +3,7 @@ from collections.abc import (
     Callable,
     Iterable,
 )
-from datetime import datetime as dt
-from datetime import timedelta
+from datetime import datetime, timedelta
 from typing import (
     Protocol,
 )
@@ -14,6 +13,7 @@ import requests
 from pydantic import BaseModel
 from sretoolbox.utils import retry
 
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.secret_reader import (
     HasSecret,
     SecretReader,
@@ -80,7 +80,7 @@ class PagerDutyApi:
     def get_pagerduty_users(
         self, resource_type: str, resource_id: str
     ) -> list[pypd.User]:
-        now = dt.utcnow()
+        now = utc_now()
 
         try:
             if resource_type == "schedule":
@@ -103,7 +103,7 @@ class PagerDutyApi:
         self.users.append(user)
         return user.email.split("@")[0]
 
-    def get_schedule_users(self, schedule_id: str, now: dt) -> list[pypd.User]:
+    def get_schedule_users(self, schedule_id: str, now: datetime) -> list[pypd.User]:
         until = now + timedelta(seconds=60)
         s = pypd.Schedule.fetch(id=schedule_id, since=now, until=until, time_zone="UTC")
         entries = s["final_schedule"]["rendered_schedule_entries"]
@@ -115,7 +115,7 @@ class PagerDutyApi:
         ]
 
     def get_escalation_policy_users(
-        self, escalation_policy_id: str, now: dt
+        self, escalation_policy_id: str, now: datetime
     ) -> list[pypd.User]:
         ep = pypd.EscalationPolicy.fetch(
             id=escalation_policy_id, since=now, until=now, time_zone="UTC"

reconcile/utils/runtime/integration.py

@@ -7,7 +7,6 @@ from dataclasses import dataclass
 from types import ModuleType
 from typing import (
     Any,
-    Generic,
     Optional,
     TypeVar,
 )
@@ -144,7 +143,7 @@ IntegrationClassTypeVar = TypeVar(
 )
 
 
-class QontractReconcileIntegration(ABC, Generic[RunParamsTypeVar]):
+class QontractReconcileIntegration[RunParamsTypeVar: RunParams](ABC):
     """
     The base class for all integrations. It defines the basic interface to interact
     with an integration and offers hook methods that allow the integration to opt

reconcile/utils/runtime/runner.py

@@ -156,7 +156,7 @@ def run_integration_cfg(run_cfg: IntegrationRunConfiguration) -> None:
         _integration_wet_run(run_cfg.integration)
 
 
-def _integration_wet_run(
+def _integration_wet_run[RunParamsTypeVar: RunParams](
     integration: QontractReconcileIntegration[RunParamsTypeVar],
 ) -> None:
     """
@@ -165,7 +165,7 @@ def _integration_wet_run(
     integration.run(False)
 
 
-def _integration_dry_run(
+def _integration_dry_run[RunParamsTypeVar: RunParams](
     integration: QontractReconcileIntegration[RunParamsTypeVar],
     desired_state_diff: DesiredStateDiff | None,
 ) -> None:

reconcile/utils/saasherder/models.py

@@ -16,6 +16,7 @@ from reconcile.gql_definitions.fragments.saas_slo_document import (
     SLODocument,
 )
 from reconcile.utils.jenkins_api import JobBuildState
+from reconcile.utils.json import json_dumps
 from reconcile.utils.oc_connection_parameters import Cluster
 from reconcile.utils.saasherder.interfaces import (
     HasParameters,
@@ -422,7 +423,7 @@ class TargetSpec:
                 elif v is False:
                     parameters[k] = "false"
                 elif any(isinstance(v, t) for t in [dict, list, tuple]):
-                    parameters[k] = json.dumps(v)
+                    parameters[k] = json_dumps(v)
         return parameters
 
     def _collect_secret_parameters(