qontract-reconcile 0.10.2.dev310__py3-none-any.whl → 0.10.2.dev439__py3-none-any.whl

reconcile/ocm_machine_pools.py

@@ -7,11 +7,7 @@ from collections.abc import Iterable, Mapping
 from enum import Enum
 from typing import Any, Self
 
-from pydantic import (
-    BaseModel,
-    Field,
-    root_validator,
-)
+from pydantic import BaseModel, Field, SerializeAsAny, model_validator
 
 from reconcile import queries
 from reconcile.gql_definitions.common.clusters import (
@@ -22,6 +18,7 @@ from reconcile.gql_definitions.common.clusters import (
 from reconcile.typed_queries.clusters import get_clusters
 from reconcile.utils.differ import diff_mappings
 from reconcile.utils.disabled_integrations import integration_is_enabled
+from reconcile.utils.json import json_dumps
 from reconcile.utils.ocm import (
     DEFAULT_OCM_MACHINE_POOL_ID,
     OCM,
@@ -61,7 +58,8 @@ class MachinePoolAutoscaling(AbstractAutoscaling):
     min_replicas: int
     max_replicas: int
 
-    @root_validator()
+    @model_validator(mode="before")
+    @classmethod
     def max_greater_min(cls, field_values: Mapping[str, Any]) -> Mapping[str, Any]:
         min_replicas = field_values.get("min_replicas")
         max_replicas = field_values.get("max_replicas")
@@ -82,7 +80,8 @@ class NodePoolAutoscaling(AbstractAutoscaling):
     min_replica: int
     max_replica: int
 
-    @root_validator()
+    @model_validator(mode="before")
+    @classmethod
     def max_greater_min(cls, field_values: Mapping[str, Any]) -> Mapping[str, Any]:
         min_replica = field_values.get("min_replica")
         max_replica = field_values.get("max_replica")
@@ -103,14 +102,15 @@ class AbstractPool(ABC, BaseModel):
     # Abstract class for machine pools, to be implemented by OSD/HyperShift classes
 
     id: str
-    replicas: int | None
-    taints: list[Mapping[str, str]] | None
-    labels: Mapping[str, str] | None
+    replicas: int | None = None
+    taints: list[Mapping[str, str]] | None = None
+    labels: Mapping[str, str] | None = None
     cluster: str
     cluster_type: ClusterType = Field(..., exclude=True)
-    autoscaling: AbstractAutoscaling | None
+    autoscaling: SerializeAsAny[AbstractAutoscaling] | None = None
 
-    @root_validator()
+    @model_validator(mode="before")
+    @classmethod
     def validate_scaling(cls, field_values: Mapping[str, Any]) -> Mapping[str, Any]:
         if field_values.get("autoscaling") and field_values.get("replicas"):
             raise ValueError("autoscaling and replicas are mutually exclusive")
@@ -154,13 +154,13 @@ class MachinePool(AbstractPool):
     instance_type: str
 
     def delete(self, ocm: OCM) -> None:
-        ocm.delete_machine_pool(self.cluster, self.dict(by_alias=True))
+        ocm.delete_machine_pool(self.cluster, self.model_dump(by_alias=True))
 
     def create(self, ocm: OCM) -> None:
-        ocm.create_machine_pool(self.cluster, self.dict(by_alias=True))
+        ocm.create_machine_pool(self.cluster, self.model_dump(by_alias=True))
 
     def update(self, ocm: OCM) -> None:
-        update_dict = self.dict(by_alias=True)
+        update_dict = self.model_dump(by_alias=True)
         # can not update instance_type
         del update_dict["instance_type"]
         if not update_dict["labels"]:
@@ -170,7 +170,10 @@ class MachinePool(AbstractPool):
         ocm.update_machine_pool(self.cluster, update_dict)
 
     def has_diff(self, pool: ClusterMachinePoolV1) -> bool:
-        if self.taints != pool.taints or self.labels != pool.labels:
+        pool_taints = (
+            [p.model_dump(by_alias=True) for p in pool.taints] if pool.taints else None
+        )
+        if self.taints != pool_taints or self.labels != pool.labels:
             logging.warning(
                 f"updating labels or taints for machine pool {pool.q_id} "
                 f"will only be applied to new Nodes"
@@ -178,7 +181,7 @@ class MachinePool(AbstractPool):
 
         return (
             self.replicas != pool.replicas
-            or self.taints != pool.taints
+            or self.taints != pool_taints
             or self.labels != pool.labels
             or self.instance_type != pool.instance_type
             or self._has_diff_autoscale(pool)
@@ -214,7 +217,7 @@ class MachinePool(AbstractPool):
             replicas=pool.replicas,
             autoscaling=autoscaling,
             instance_type=pool.instance_type,
-            taints=[p.dict(by_alias=True) for p in pool.taints or []],
+            taints=[p.model_dump(by_alias=True) for p in pool.taints or []],
             labels=pool.labels,
             cluster=cluster,
             cluster_type=cluster_type,
@@ -232,14 +235,14 @@ class NodePool(AbstractPool):
     subnet: str | None
 
     def delete(self, ocm: OCM) -> None:
-        ocm.delete_node_pool(self.cluster, self.dict(by_alias=True))
+        ocm.delete_node_pool(self.cluster, self.model_dump(by_alias=True))
 
     def create(self, ocm: OCM) -> None:
-        spec = self.dict(by_alias=True)
+        spec = self.model_dump(by_alias=True)
         ocm.create_node_pool(self.cluster, spec)
 
     def update(self, ocm: OCM) -> None:
-        update_dict = self.dict(by_alias=True)
+        update_dict = self.model_dump(by_alias=True)
         # can not update instance_type
         del update_dict["aws_node_pool"]
         # can not update subnet
@@ -251,7 +254,10 @@ class NodePool(AbstractPool):
         ocm.update_node_pool(self.cluster, update_dict)
 
     def has_diff(self, pool: ClusterMachinePoolV1) -> bool:
-        if self.taints != pool.taints or self.labels != pool.labels:
+        pool_taints = (
+            [p.model_dump(by_alias=True) for p in pool.taints] if pool.taints else None
+        )
+        if self.taints != pool_taints or self.labels != pool.labels:
             logging.warning(
                 f"updating labels or taints for node pool {pool.q_id} "
                 f"will only be applied to new Nodes"
@@ -259,7 +265,7 @@ class NodePool(AbstractPool):
 
         return (
             self.replicas != pool.replicas
-            or self.taints != pool.taints
+            or self.taints != pool_taints
             or self.labels != pool.labels
             or self.aws_node_pool.instance_type != pool.instance_type
             or self.subnet != pool.subnet
@@ -297,7 +303,7 @@ class NodePool(AbstractPool):
             aws_node_pool=AWSNodePool(
                 instance_type=pool.instance_type,
             ),
-            taints=[p.dict(by_alias=True) for p in pool.taints or []],
+            taints=[p.model_dump(by_alias=True) for p in pool.taints or []],
             labels=pool.labels,
             subnet=pool.subnet,
             cluster=cluster,
@@ -312,7 +318,7 @@ class PoolHandler(BaseModel):
     pool: AbstractPool
 
     def act(self, dry_run: bool, ocm: OCM) -> None:
-        logging.info(f"{self.action} {self.pool.dict(by_alias=True)}")
+        logging.info(f"{self.action} {self.pool.model_dump(by_alias=True)}")
         if dry_run:
             return
 
@@ -469,7 +475,7 @@ def calculate_diff(
         if invalid_diff:
             errors.append(
                 InvalidUpdateError(
-                    f"can not update {invalid_diff} for existing machine pool on cluster {cluster_name}, CURRENT: {diff_pair.current.json()}, DESIRED: {diff_pair.desired.json()}"
+                    f"can not update {invalid_diff} for existing machine pool on cluster {cluster_name}, CURRENT: {json_dumps(diff_pair.current)}, DESIRED: {json_dumps(diff_pair.desired)}"
                 )
             )
         else:
@@ -531,7 +537,7 @@ def run(dry_run: bool) -> None:
 
     settings = queries.get_app_interface_settings()
     cluster_like_objects = [
-        cluster.dict(by_alias=True) for cluster in filtered_clusters
+        cluster.model_dump(by_alias=True) for cluster in filtered_clusters
     ]
     ocm_map = OCMMap(
         clusters=cluster_like_objects,

reconcile/openshift_base.py

@@ -29,10 +29,14 @@ from reconcile.utils import (
     metrics,
 )
 from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
+from reconcile.utils.differ import DiffPair
 from reconcile.utils.oc import (
+    POD_RECYCLE_SUPPORTED_OWNER_KINDS,
+    AmbiguousResourceTypeError,
     DeploymentFieldIsImmutableError,
     FieldIsImmutableError,
     InvalidValueApplyError,
+    KindNotFoundError,
     MayNotChangeOnceSetError,
     MetaDataAnnotationsTooLongApplyError,
     OC_Map,
@@ -60,6 +64,10 @@ AUTH_METHOD_USER_KEY = {
     "oidc": "org_username",
     "rhidp": "org_username",
 }
+RECYCLE_POD_ANNOTATIONS = [
+    "kubectl.kubernetes.io/restartedAt",
+    "openshift.openshift.io/restartedAt",
+]
 
 
 class ValidationError(Exception):
@@ -128,6 +136,29 @@ class ClusterMap(Protocol):
     ) -> list[str]: ...
 
 
+def validate_managed_resource_types(
+    oc: OCCli,
+    managed_resource_types: Iterable[str],
+    managed_resource_names: Iterable[Mapping[str, Any]],
+    cluster_scope_resource_validation: bool,
+) -> None:
+    """Validate the managed resource types."""
+    managed_resources = [
+        managed_resource_name["resource"]
+        for managed_resource_name in managed_resource_names
+    ]
+    for managed_resource_type in managed_resource_types:
+        # The k8s kind must be supported by the cluster
+        resource = oc.get_api_resource(managed_resource_type)
+
+        if cluster_scope_resource_validation and not resource.namespaced:
+            # cluster-scoped resources must be use managedResourceNames!
+            if managed_resource_type not in managed_resources:
+                raise ValidationError(
+                    f"Cluster-scoped resource {managed_resource_type} must be managed by name only. Please use 'managedResourceNames' field to specify the names of the resources to manage."
+                )
+
+
 def init_specs_to_fetch(
     ri: ResourceInventory,
     oc_map: ClusterMap,
@@ -136,6 +167,7 @@ def init_specs_to_fetch(
     override_managed_types: Iterable[str] | None = None,
     managed_types_key: str = "managedResourceTypes",
     cluster_admin: bool = False,
+    cluster_scope_resource_validation: bool = False,
 ) -> list[StateSpec]:
     state_specs: list[StateSpec] = []
 
@@ -163,9 +195,27 @@ def init_specs_to_fetch(
                 logging.log(level=ex.log_level, msg=ex.message)
                 continue
 
+            managed_resource_names = namespace_info.get("managedResourceNames") or []
+            try:
+                validate_managed_resource_types(
+                    oc,
+                    managed_types,
+                    managed_resource_names,
+                    cluster_scope_resource_validation=cluster_scope_resource_validation,
+                )
+            except KindNotFoundError:
+                # We must allow kinds that are not supported by the cluster because:
+                # 1. We install CRD with an operator in the same MR
+                # 2. SAAS files initialize the namespace objects with managedResourceTypes from the SAAS file
+                #    and we can't expect that all of those are valid for all clusters
+                pass
+            except (AmbiguousResourceTypeError, ValidationError) as e:
+                ri.register_error()
+                logging.error(f"[{cluster}/{namespace_info['name']}] {e}")
+                continue
+
             namespace = namespace_info["name"]
             # These may exit but have a value of None
-            managed_resource_names = namespace_info.get("managedResourceNames") or []
             managed_resource_type_overrides = (
                 namespace_info.get("managedResourceTypeOverrides") or []
             )
@@ -315,7 +365,6 @@ def populate_current_state(
 
             if caller and openshift_resource.caller != caller:
                 continue
-
             ri.add_current(
                 spec.cluster,
                 spec.namespace,
@@ -341,6 +390,7 @@ def fetch_current_state(
     cluster_admin: bool = False,
     caller: str | None = None,
     init_projects: bool = False,
+    cluster_scope_resource_validation: bool = False,
 ) -> tuple[ResourceInventory, OC_Map]:
     ri = ResourceInventory()
     settings = queries.get_app_interface_settings()
@@ -363,6 +413,7 @@ def fetch_current_state(
         clusters=clusters,
         override_managed_types=override_managed_types,
         cluster_admin=cluster_admin,
+        cluster_scope_resource_validation=cluster_scope_resource_validation,
     )
     threaded.run(
         populate_current_state,
@@ -535,11 +586,15 @@ def apply(
             # take care of the resize ourselves.
             # ref: https://github.com/kubernetes/enhancements/pull/2842
             if resize_required:
+                if desired_storage is None:
+                    raise ValidationError(
+                        "Resource body does not contain storage information."
+                    ) from None
                 logging.info(["resizing_pvcs", cluster, namespace, owned_pvc_names])
                 oc.resize_pvcs(namespace, owned_pvc_names, desired_storage)
 
     if recycle_pods:
-        oc.recycle_pods(dry_run, namespace, resource_type, resource)
+        oc.recycle_pods(dry_run, namespace, resource)
 
 
 def create(
@@ -783,10 +838,56 @@ def handle_identical_resources(
     return actions
 
 
+def patch_desired_resource_for_recycle_annotations(
+    desired: OR,
+    current: OR,
+) -> OR:
+    """
+    Patch desired resource with recycle annotations to pod template from current resource.
+    This is to avoid full pods recycle when changes are not affecting pod template.
+    Note desired annotations can override current annotations.
+    For example, if desired resource has kubectl.kubernetes.io/restartedAt defined,
+    it will be used instead of current resource annotation.
+
+    Args:
+        desired: desired resource
+        current: current resource
+
+    Returns:
+        patched desired resource
+    """
+    if current.kind not in POD_RECYCLE_SUPPORTED_OWNER_KINDS:
+        return desired
+
+    current_annotations = (
+        current.body.get("spec", {})
+        .get("template", {})
+        .get("metadata", {})
+        .get("annotations")
+        or {}
+    )
+    patch_annotations = {
+        k: value
+        for k in RECYCLE_POD_ANNOTATIONS
+        if (value := current_annotations.get(k))
+    }
+    if patch_annotations:
+        desired_annotations = (
+            desired.body.setdefault("spec", {})
+            .setdefault("template", {})
+            .setdefault("metadata", {})
+            .setdefault("annotations", {})
+        )
+        desired.body["spec"]["template"]["metadata"]["annotations"] = (
+            patch_annotations | desired_annotations
+        )
+    return desired
+
+
 def handle_modified_resources(
     oc_map: ClusterMap,
     ri: ResourceInventory,
-    modified_resources: Mapping[Any, Any],
+    modified_resources: Mapping[str, DiffPair[OR, OR]],
     cluster: str,
     namespace: str,
     resource_type: str,
@@ -982,6 +1083,12 @@ def _realize_resource_data_3way_diff(
     if options.enable_deletion and options.override_enable_deletion is False:
         options.enable_deletion = False
 
+    for k in data["current"].keys() & data["desired"].keys():
+        patch_desired_resource_for_recycle_annotations(
+            desired=data["desired"][k],
+            current=data["current"][k],
+        )
+
     diff_result = differ.diff_mappings(
         data["current"], data["desired"], equal=three_way_diff_using_hash
     )
@@ -1095,11 +1202,11 @@ def _validate_resources_used_exist(
     )
     for used_name, used_keys in used_resources.items():
         # perhaps used resource is deployed together with the using resource?
-        resource = ri.get_desired(cluster, namespace, used_kind, used_name)
+        desired_resource = ri.get_desired(cluster, namespace, used_kind, used_name)
         # if not, perhaps it's a secret that will be created from a Service's
         # serving-cert that is deployed along with the using resource?
         # lets iterate through all resources and find Services that have the annotation
-        if not resource and used_kind == "Secret":
+        if not desired_resource and used_kind == "Secret":
             # consider only Service resources that are in the same cluster & namespace
             service_resources = []
             for cname, nname, restype, res in ri:
@@ -1123,12 +1230,12 @@ def _validate_resources_used_exist(
                 }:
                     # found a match. we assume the serving cert secret will
                     # be present at some point soon after the Service is deployed
-                    resource = service
+                    desired_resource = service
                     break
 
-        if resource:
+        if desired_resource:
             # get the body to match with the possible result from oc.get
-            resource = resource.body
+            resource = desired_resource.body
         else:
             # no. perhaps used resource exists in the namespace?
             resource = oc.get(
@@ -1360,6 +1467,11 @@ class HasOpenShiftResources(Protocol):
     openshift_resources: list | None
 
 
+@runtime_checkable
+class HasOpenShiftResourcesRequired(Protocol):
+    openshift_resources: list
+
+
 @runtime_checkable
 class HasOpenshiftServiceAccountTokens(Protocol):
     openshift_service_account_tokens: list | None
@@ -1368,7 +1480,7 @@ class HasOpenshiftServiceAccountTokens(Protocol):
 @runtime_checkable
 class HasSharedResourcesOpenShiftResources(Protocol):
     @property
-    def shared_resources(self) -> Sequence[HasOpenShiftResources] | None: ...
+    def shared_resources(self) -> Sequence[HasOpenShiftResourcesRequired] | None: ...
 
 
 @runtime_checkable

reconcile/openshift_cluster_bots.py

@@ -5,7 +5,6 @@ import subprocess
 import sys
 import tempfile
 import urllib.request
-from typing import cast
 from urllib.error import URLError
 
 from pydantic import BaseModel
@@ -17,6 +16,7 @@ from reconcile.gql_definitions.openshift_cluster_bots.clusters import ClusterV1
 from reconcile.status import ExitCodes
 from reconcile.utils import gql
 from reconcile.utils.disabled_integrations import integration_is_enabled
+from reconcile.utils.json import json_dumps
 from reconcile.utils.mr import clusters_updates
 from reconcile.utils.ocm import OCM, OCMMap
 from reconcile.utils.openshift_resource import (
@@ -24,7 +24,7 @@ from reconcile.utils.openshift_resource import (
     QONTRACT_ANNOTATION_INTEGRATION_VERSION,
 )
 from reconcile.utils.semver_helper import make_semver
-from reconcile.utils.vault import VaultClient, _VaultClient
+from reconcile.utils.vault import VaultClient
 
 QONTRACT_INTEGRATION = "openshift-cluster-bots"
 QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
@@ -103,7 +103,7 @@ def oc(
 
 def oc_apply(kubeconfig: str, namespace: str, items: list[dict]) -> None:
     for item in items:
-        stdin = json.dumps(item).encode()
+        stdin = json_dumps(item).encode()
         oc(kubeconfig, namespace, ["apply", "-f", "-"], stdin)
 
 
@@ -238,7 +238,7 @@ def create_cluster_bots(
 def update_vault(
     cluster: ClusterV1, config: Config, token: str, admin_token: str | None
 ) -> None:
-    vault = cast("_VaultClient", VaultClient())
+    vault = VaultClient.get_instance()
     vault.write(
         {
             "path": vault_secret(cluster, config, cluster_admin=False)["path"],
@@ -301,7 +301,7 @@ def filter_clusters(clusters: list[ClusterV1]) -> list[ClusterV1]:
 
 def get_ocm_map(clusters: list[ClusterV1]) -> OCMMap:
     settings = queries.get_app_interface_settings()
-    clusters_info = [c.dict(by_alias=True) for c in clusters]
+    clusters_info = [c.model_dump(by_alias=True) for c in clusters]
     return OCMMap(
         settings=settings,
         clusters=clusters_info,

reconcile/openshift_groups.py

@@ -243,10 +243,15 @@ def act(diff: Mapping[str, str | None], oc_map: ClusterMap) -> None:
         return None
 
     if action == "create_group":
+        assert group  # make mypy happy
         oc.create_group(group)
     elif action == "add_user_to_group":
+        assert group  # make mypy happy
+        assert user  # make mypy happy
         oc.add_user_to_group(group, user)
     elif action == "del_user_from_group":
+        assert group  # make mypy happy
+        assert user  # make mypy happy
         oc.del_user_from_group(group, user)
     elif action == "delete_group":
         logging.debug("skipping group deletion")

reconcile/openshift_limitranges.py

@@ -39,7 +39,7 @@ def construct_resources(
         # Get the linked limitRanges schema settings
         limitranges = namespace.get("limitRanges", {})
 
-        body: MutableMapping[str, Any] = {
+        body: dict[str, Any] = {
             "apiVersion": "v1",
             "kind": "LimitRange",
             "metadata": {

reconcile/openshift_namespace_labels.py

@@ -229,7 +229,7 @@ def get_gql_namespaces_in_shard() -> list[NamespaceV1]:
     return [
         ns
         for ns in all_namespaces
-        if not ob.is_namespace_deleted(ns.dict(by_alias=True))
+        if not ob.is_namespace_deleted(ns.model_dump(by_alias=True))
         and is_in_shard(f"{ns.cluster.name}/{ns.name}")
     ]