qontract-reconcile 0.10.2.dev394__py3-none-any.whl → 0.10.2.dev427__py3-none-any.whl

reconcile/ocm/types.py

@@ -1,10 +1,6 @@
 from __future__ import annotations
 
-from pydantic import (
-    BaseModel,
-    Extra,
-    Field,
-)
+from pydantic import BaseModel, Field, field_validator
 
 
 class OCMClusterAutoscale(BaseModel):
@@ -13,100 +9,82 @@ class OCMClusterAutoscale(BaseModel):
 
 
 class OCMClusterNetwork(BaseModel):
-    type: str | None
+    type: str | None = None
     vpc: str
     service: str
     pod: str
 
 
-class OCMClusterSpec(BaseModel):
-    autoscale: OCMClusterAutoscale | None
+class OCMClusterSpec(BaseModel, extra="forbid"):
+    autoscale: OCMClusterAutoscale | None = None
     channel: str
-    disable_user_workload_monitoring: bool | None
-    external_id: str | None
-    id: str | None
-    instance_type: str | None
-    multi_az: bool | None
-    nodes: int | None
+    disable_user_workload_monitoring: bool | None = None
+    external_id: str | None = None
+    id: str | None = None
+    instance_type: str | None = None
+    multi_az: bool | None = None
+    nodes: int | None = None
     private: bool
     product: str
     provider: str
-    provision_shard_id: str | None
+    provision_shard_id: str | None = None
     region: str
-    initial_version: str | None
+    initial_version: str | None = None
     version: str
-    hypershift: bool | None
-    fips: bool | None
+    hypershift: bool | None = None
+    fips: bool = False
 
-    class Config:
-        extra = Extra.forbid
+    @field_validator("fips", mode="before")
+    @classmethod
+    def set_fips_default(cls, v: bool | None) -> bool:
+        return v or False
 
 
-class OSDClusterSpec(OCMClusterSpec):
+class OSDClusterSpec(OCMClusterSpec, extra="forbid"):
     load_balancers: int
     storage: int
 
-    class Config:
-        extra = Extra.forbid
 
-
-class ROSAOcmAwsStsAttrs(BaseModel):
+class ROSAOcmAwsStsAttrs(BaseModel, extra="forbid"):
     installer_role_arn: str
     support_role_arn: str
-    controlplane_role_arn: str | None
+    controlplane_role_arn: str | None = None
     worker_role_arn: str
 
-    class Config:
-        extra = Extra.forbid
-
 
-class ROSAOcmAwsAttrs(BaseModel):
+class ROSAOcmAwsAttrs(BaseModel, extra="forbid"):
     creator_role_arn: str
-    sts: ROSAOcmAwsStsAttrs | None
-
-    class Config:
-        extra = Extra.forbid
+    sts: ROSAOcmAwsStsAttrs | None = None
 
 
-class ROSAClusterAWSAccount(BaseModel):
+class ROSAClusterAWSAccount(BaseModel, extra="forbid"):
     uid: str
-    rosa: ROSAOcmAwsAttrs | None
-    billing_account_id: str | None
+    rosa: ROSAOcmAwsAttrs | None = None
+    billing_account_id: str | None = None
 
-    class Config:
-        extra = Extra.forbid
 
-
-class ROSAClusterSpec(OCMClusterSpec):
+class ROSAClusterSpec(OCMClusterSpec, extra="forbid"):
     account: ROSAClusterAWSAccount
-    subnet_ids: list[str] | None
-    availability_zones: list[str] | None
-    oidc_endpoint_url: str | None
-
-    class Config:
-        extra = Extra.forbid
+    subnet_ids: list[str] | None = None
+    availability_zones: list[str] | None = None
+    oidc_endpoint_url: str | None = None
 
 
 class ClusterMachinePool(BaseModel):
     id: str
     instance_type: str
-    replicas: int | None
-    autoscale: OCMClusterAutoscale | None
+    replicas: int | None = None
+    autoscale: OCMClusterAutoscale | None = None
 
 
-class OCMSpec(BaseModel):
-    path: str | None
+class OCMSpec(BaseModel, validate_by_name=True, validate_by_alias=True):
+    path: str | None = None
     spec: OSDClusterSpec | ROSAClusterSpec | OCMClusterSpec
     machine_pools: list[ClusterMachinePool] = Field(
         default_factory=list, alias="machinePools"
     )
     network: OCMClusterNetwork
-    domain: str | None
+    domain: str | None = None
     server_url: str = Field("", alias="serverUrl")
     console_url: str = Field("", alias="consoleUrl")
     elb_fqdn: str = Field("", alias="elbFQDN")
-
-    class Config:
-        smart_union = True
-        # This is need to populate by either console_url or consoleUrl, for instance
-        allow_population_by_field_name = True

reconcile/ocm_aws_infrastructure_access.py

@@ -103,7 +103,7 @@ def fetch_desired_state(clusters: Iterable[Mapping[str, Any]]) -> list[dict[str,
     namespaces = get_namespaces()
     for namespace_info in namespaces:
         specs = get_external_resource_specs(
-            namespace_info.dict(by_alias=True), provision_provider=PROVIDER_AWS
+            namespace_info.model_dump(by_alias=True), provision_provider=PROVIDER_AWS
         )
         for spec in specs:
             if spec.provider != "aws-iam-service-account":

reconcile/ocm_clusters.py

@@ -248,13 +248,13 @@ def get_cluster_ocm_update_spec(
     if not desired_spec.network.type:
         desired_spec.network.type = "OVNKubernetes"
 
-    cspec = current_spec.spec.dict()
-    cspec[ocmmod.SPEC_ATTR_NETWORK] = current_spec.network.dict(
+    cspec = current_spec.spec.model_dump()
+    cspec[ocmmod.SPEC_ATTR_NETWORK] = current_spec.network.model_dump(
         exclude={IGNORE_NETWORK_TYPE_ATTR}
     )
 
-    dspec = desired_spec.spec.dict()
-    dspec[ocmmod.SPEC_ATTR_NETWORK] = desired_spec.network.dict(
+    dspec = desired_spec.spec.model_dump()
+    dspec[ocmmod.SPEC_ATTR_NETWORK] = desired_spec.network.model_dump(
         exclude={IGNORE_NETWORK_TYPE_ATTR}
     )
 

reconcile/ocm_labels/integration.py

@@ -4,7 +4,7 @@ import logging
 from typing import TYPE_CHECKING, Any
 
 from deepdiff import DeepHash
-from pydantic import validator
+from pydantic import field_validator
 
 from reconcile.aus.aus_label_source import (
     init_aus_cluster_label_source,
@@ -62,7 +62,8 @@ class OcmLabelsIntegrationParams(PydanticRunParams):
     managed_label_prefixes: list[str] = []
     ignored_label_prefixes: list[str] = []
 
-    @validator("managed_label_prefixes", "ignored_label_prefixes")
+    @field_validator("managed_label_prefixes", "ignored_label_prefixes")
+    @classmethod
     def must_end_with_dot(cls, v: list[str]) -> list[str]:
         return [prefix + "." if not prefix.endswith(".") else prefix for prefix in v]
 

reconcile/ocm_machine_pools.py

@@ -7,11 +7,7 @@ from collections.abc import Iterable, Mapping
 from enum import Enum
 from typing import Any, Self
 
-from pydantic import (
-    BaseModel,
-    Field,
-    root_validator,
-)
+from pydantic import BaseModel, Field, SerializeAsAny, model_validator
 
 from reconcile import queries
 from reconcile.gql_definitions.common.clusters import (
@@ -22,6 +18,7 @@ from reconcile.gql_definitions.common.clusters import (
 from reconcile.typed_queries.clusters import get_clusters
 from reconcile.utils.differ import diff_mappings
 from reconcile.utils.disabled_integrations import integration_is_enabled
+from reconcile.utils.json import json_dumps
 from reconcile.utils.ocm import (
     DEFAULT_OCM_MACHINE_POOL_ID,
     OCM,
@@ -61,7 +58,8 @@ class MachinePoolAutoscaling(AbstractAutoscaling):
     min_replicas: int
     max_replicas: int
 
-    @root_validator()
+    @model_validator(mode="before")
+    @classmethod
     def max_greater_min(cls, field_values: Mapping[str, Any]) -> Mapping[str, Any]:
         min_replicas = field_values.get("min_replicas")
         max_replicas = field_values.get("max_replicas")
@@ -82,7 +80,8 @@ class NodePoolAutoscaling(AbstractAutoscaling):
     min_replica: int
     max_replica: int
 
-    @root_validator()
+    @model_validator(mode="before")
+    @classmethod
     def max_greater_min(cls, field_values: Mapping[str, Any]) -> Mapping[str, Any]:
         min_replica = field_values.get("min_replica")
         max_replica = field_values.get("max_replica")
@@ -103,14 +102,15 @@ class AbstractPool(ABC, BaseModel):
     # Abstract class for machine pools, to be implemented by OSD/HyperShift classes
 
     id: str
-    replicas: int | None
-    taints: list[Mapping[str, str]] | None
-    labels: Mapping[str, str] | None
+    replicas: int | None = None
+    taints: list[Mapping[str, str]] | None = None
+    labels: Mapping[str, str] | None = None
     cluster: str
     cluster_type: ClusterType = Field(..., exclude=True)
-    autoscaling: AbstractAutoscaling | None
+    autoscaling: SerializeAsAny[AbstractAutoscaling] | None = None
 
-    @root_validator()
+    @model_validator(mode="before")
+    @classmethod
     def validate_scaling(cls, field_values: Mapping[str, Any]) -> Mapping[str, Any]:
         if field_values.get("autoscaling") and field_values.get("replicas"):
             raise ValueError("autoscaling and replicas are mutually exclusive")
@@ -154,13 +154,13 @@ class MachinePool(AbstractPool):
     instance_type: str
 
     def delete(self, ocm: OCM) -> None:
-        ocm.delete_machine_pool(self.cluster, self.dict(by_alias=True))
+        ocm.delete_machine_pool(self.cluster, self.model_dump(by_alias=True))
 
     def create(self, ocm: OCM) -> None:
-        ocm.create_machine_pool(self.cluster, self.dict(by_alias=True))
+        ocm.create_machine_pool(self.cluster, self.model_dump(by_alias=True))
 
     def update(self, ocm: OCM) -> None:
-        update_dict = self.dict(by_alias=True)
+        update_dict = self.model_dump(by_alias=True)
         # can not update instance_type
         del update_dict["instance_type"]
         if not update_dict["labels"]:
@@ -170,7 +170,10 @@ class MachinePool(AbstractPool):
         ocm.update_machine_pool(self.cluster, update_dict)
 
     def has_diff(self, pool: ClusterMachinePoolV1) -> bool:
-        if self.taints != pool.taints or self.labels != pool.labels:
+        pool_taints = (
+            [p.model_dump(by_alias=True) for p in pool.taints] if pool.taints else None
+        )
+        if self.taints != pool_taints or self.labels != pool.labels:
             logging.warning(
                 f"updating labels or taints for machine pool {pool.q_id} "
                 f"will only be applied to new Nodes"
@@ -178,7 +181,7 @@ class MachinePool(AbstractPool):
 
         return (
             self.replicas != pool.replicas
-            or self.taints != pool.taints
+            or self.taints != pool_taints
             or self.labels != pool.labels
             or self.instance_type != pool.instance_type
             or self._has_diff_autoscale(pool)
@@ -214,7 +217,7 @@ class MachinePool(AbstractPool):
             replicas=pool.replicas,
             autoscaling=autoscaling,
             instance_type=pool.instance_type,
-            taints=[p.dict(by_alias=True) for p in pool.taints or []],
+            taints=[p.model_dump(by_alias=True) for p in pool.taints or []],
             labels=pool.labels,
             cluster=cluster,
             cluster_type=cluster_type,
@@ -232,14 +235,14 @@ class NodePool(AbstractPool):
     subnet: str | None
 
     def delete(self, ocm: OCM) -> None:
-        ocm.delete_node_pool(self.cluster, self.dict(by_alias=True))
+        ocm.delete_node_pool(self.cluster, self.model_dump(by_alias=True))
 
     def create(self, ocm: OCM) -> None:
-        spec = self.dict(by_alias=True)
+        spec = self.model_dump(by_alias=True)
         ocm.create_node_pool(self.cluster, spec)
 
     def update(self, ocm: OCM) -> None:
-        update_dict = self.dict(by_alias=True)
+        update_dict = self.model_dump(by_alias=True)
         # can not update instance_type
         del update_dict["aws_node_pool"]
         # can not update subnet
@@ -251,7 +254,10 @@ class NodePool(AbstractPool):
         ocm.update_node_pool(self.cluster, update_dict)
 
     def has_diff(self, pool: ClusterMachinePoolV1) -> bool:
-        if self.taints != pool.taints or self.labels != pool.labels:
+        pool_taints = (
+            [p.model_dump(by_alias=True) for p in pool.taints] if pool.taints else None
+        )
+        if self.taints != pool_taints or self.labels != pool.labels:
             logging.warning(
                 f"updating labels or taints for node pool {pool.q_id} "
                 f"will only be applied to new Nodes"
@@ -259,7 +265,7 @@ class NodePool(AbstractPool):
 
         return (
             self.replicas != pool.replicas
-            or self.taints != pool.taints
+            or self.taints != pool_taints
             or self.labels != pool.labels
             or self.aws_node_pool.instance_type != pool.instance_type
             or self.subnet != pool.subnet
@@ -297,7 +303,7 @@ class NodePool(AbstractPool):
             aws_node_pool=AWSNodePool(
                 instance_type=pool.instance_type,
             ),
-            taints=[p.dict(by_alias=True) for p in pool.taints or []],
+            taints=[p.model_dump(by_alias=True) for p in pool.taints or []],
             labels=pool.labels,
             subnet=pool.subnet,
             cluster=cluster,
@@ -312,7 +318,7 @@ class PoolHandler(BaseModel):
     pool: AbstractPool
 
     def act(self, dry_run: bool, ocm: OCM) -> None:
-        logging.info(f"{self.action} {self.pool.dict(by_alias=True)}")
+        logging.info(f"{self.action} {self.pool.model_dump(by_alias=True)}")
         if dry_run:
             return
 
@@ -469,7 +475,7 @@ def calculate_diff(
         if invalid_diff:
             errors.append(
                 InvalidUpdateError(
-                    f"can not update {invalid_diff} for existing machine pool on cluster {cluster_name}, CURRENT: {diff_pair.current.json()}, DESIRED: {diff_pair.desired.json()}"
+                    f"can not update {invalid_diff} for existing machine pool on cluster {cluster_name}, CURRENT: {json_dumps(diff_pair.current)}, DESIRED: {json_dumps(diff_pair.desired)}"
                 )
             )
         else:
@@ -531,7 +537,7 @@ def run(dry_run: bool) -> None:
 
     settings = queries.get_app_interface_settings()
     cluster_like_objects = [
-        cluster.dict(by_alias=True) for cluster in filtered_clusters
+        cluster.model_dump(by_alias=True) for cluster in filtered_clusters
     ]
     ocm_map = OCMMap(
         clusters=cluster_like_objects,

reconcile/openshift_base.py

@@ -29,10 +29,14 @@ from reconcile.utils import (
     metrics,
 )
 from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
+from reconcile.utils.differ import DiffPair
 from reconcile.utils.oc import (
+    POD_RECYCLE_SUPPORTED_OWNER_KINDS,
+    AmbiguousResourceTypeError,
     DeploymentFieldIsImmutableError,
     FieldIsImmutableError,
     InvalidValueApplyError,
+    KindNotFoundError,
     MayNotChangeOnceSetError,
     MetaDataAnnotationsTooLongApplyError,
     OC_Map,
@@ -60,6 +64,10 @@ AUTH_METHOD_USER_KEY = {
     "oidc": "org_username",
     "rhidp": "org_username",
 }
+RECYCLE_POD_ANNOTATIONS = [
+    "kubectl.kubernetes.io/restartedAt",
+    "openshift.openshift.io/restartedAt",
+]
 
 
 class ValidationError(Exception):
@@ -128,6 +136,29 @@ class ClusterMap(Protocol):
     ) -> list[str]: ...
 
 
+def validate_managed_resource_types(
+    oc: OCCli,
+    managed_resource_types: Iterable[str],
+    managed_resource_names: Iterable[Mapping[str, Any]],
+    cluster_scope_resource_validation: bool,
+) -> None:
+    """Validate the managed resource types."""
+    managed_resources = [
+        managed_resource_name["resource"]
+        for managed_resource_name in managed_resource_names
+    ]
+    for managed_resource_type in managed_resource_types:
+        # The k8s kind must be supported by the cluster
+        resource = oc.get_api_resource(managed_resource_type)
+
+        if cluster_scope_resource_validation and not resource.namespaced:
+            # cluster-scoped resources must be use managedResourceNames!
+            if managed_resource_type not in managed_resources:
+                raise ValidationError(
+                    f"Cluster-scoped resource {managed_resource_type} must be managed by name only. Please use 'managedResourceNames' field to specify the names of the resources to manage."
+                )
+
+
 def init_specs_to_fetch(
     ri: ResourceInventory,
     oc_map: ClusterMap,
@@ -136,6 +167,7 @@ def init_specs_to_fetch(
     override_managed_types: Iterable[str] | None = None,
     managed_types_key: str = "managedResourceTypes",
     cluster_admin: bool = False,
+    cluster_scope_resource_validation: bool = False,
 ) -> list[StateSpec]:
     state_specs: list[StateSpec] = []
 
@@ -163,9 +195,27 @@ def init_specs_to_fetch(
                 logging.log(level=ex.log_level, msg=ex.message)
                 continue
 
+            managed_resource_names = namespace_info.get("managedResourceNames") or []
+            try:
+                validate_managed_resource_types(
+                    oc,
+                    managed_types,
+                    managed_resource_names,
+                    cluster_scope_resource_validation=cluster_scope_resource_validation,
+                )
+            except KindNotFoundError:
+                # We must allow kinds that are not supported by the cluster because:
+                # 1. We install CRD with an operator in the same MR
+                # 2. SAAS files initialize the namespace objects with managedResourceTypes from the SAAS file
+                #    and we can't expect that all of those are valid for all clusters
+                pass
+            except (AmbiguousResourceTypeError, ValidationError) as e:
+                ri.register_error()
+                logging.error(f"[{cluster}/{namespace_info['name']}] {e}")
+                continue
+
             namespace = namespace_info["name"]
             # These may exit but have a value of None
-            managed_resource_names = namespace_info.get("managedResourceNames") or []
             managed_resource_type_overrides = (
                 namespace_info.get("managedResourceTypeOverrides") or []
             )
@@ -340,6 +390,7 @@ def fetch_current_state(
     cluster_admin: bool = False,
     caller: str | None = None,
     init_projects: bool = False,
+    cluster_scope_resource_validation: bool = False,
 ) -> tuple[ResourceInventory, OC_Map]:
     ri = ResourceInventory()
     settings = queries.get_app_interface_settings()
@@ -362,6 +413,7 @@ def fetch_current_state(
         clusters=clusters,
         override_managed_types=override_managed_types,
         cluster_admin=cluster_admin,
+        cluster_scope_resource_validation=cluster_scope_resource_validation,
     )
     threaded.run(
         populate_current_state,
@@ -542,7 +594,7 @@ def apply(
                 oc.resize_pvcs(namespace, owned_pvc_names, desired_storage)
 
     if recycle_pods:
-        oc.recycle_pods(dry_run, namespace, resource_type, resource)
+        oc.recycle_pods(dry_run, namespace, resource)
 
 
 def create(
@@ -786,10 +838,56 @@ def handle_identical_resources(
     return actions
 
 
+def patch_desired_resource_for_recycle_annotations(
+    desired: OR,
+    current: OR,
+) -> OR:
+    """
+    Patch desired resource with recycle annotations to pod template from current resource.
+    This is to avoid full pods recycle when changes are not affecting pod template.
+    Note desired annotations can override current annotations.
+    For example, if desired resource has kubectl.kubernetes.io/restartedAt defined,
+    it will be used instead of current resource annotation.
+
+    Args:
+        desired: desired resource
+        current: current resource
+
+    Returns:
+        patched desired resource
+    """
+    if current.kind not in POD_RECYCLE_SUPPORTED_OWNER_KINDS:
+        return desired
+
+    current_annotations = (
+        current.body.get("spec", {})
+        .get("template", {})
+        .get("metadata", {})
+        .get("annotations")
+        or {}
+    )
+    patch_annotations = {
+        k: value
+        for k in RECYCLE_POD_ANNOTATIONS
+        if (value := current_annotations.get(k))
+    }
+    if patch_annotations:
+        desired_annotations = (
+            desired.body.setdefault("spec", {})
+            .setdefault("template", {})
+            .setdefault("metadata", {})
+            .setdefault("annotations", {})
+        )
+        desired.body["spec"]["template"]["metadata"]["annotations"] = (
+            patch_annotations | desired_annotations
+        )
+    return desired
+
+
 def handle_modified_resources(
     oc_map: ClusterMap,
     ri: ResourceInventory,
-    modified_resources: Mapping[Any, Any],
+    modified_resources: Mapping[str, DiffPair[OR, OR]],
     cluster: str,
     namespace: str,
     resource_type: str,
@@ -985,6 +1083,12 @@ def _realize_resource_data_3way_diff(
     if options.enable_deletion and options.override_enable_deletion is False:
         options.enable_deletion = False
 
+    for k in data["current"].keys() & data["desired"].keys():
+        patch_desired_resource_for_recycle_annotations(
+            desired=data["desired"][k],
+            current=data["current"][k],
+        )
+
     diff_result = differ.diff_mappings(
         data["current"], data["desired"], equal=three_way_diff_using_hash
     )
@@ -1363,6 +1467,11 @@ class HasOpenShiftResources(Protocol):
     openshift_resources: list | None
 
 
+@runtime_checkable
+class HasOpenShiftResourcesRequired(Protocol):
+    openshift_resources: list
+
+
 @runtime_checkable
 class HasOpenshiftServiceAccountTokens(Protocol):
     openshift_service_account_tokens: list | None
@@ -1371,7 +1480,7 @@ class HasOpenshiftServiceAccountTokens(Protocol):
 @runtime_checkable
 class HasSharedResourcesOpenShiftResources(Protocol):
     @property
-    def shared_resources(self) -> Sequence[HasOpenShiftResources] | None: ...
+    def shared_resources(self) -> Sequence[HasOpenShiftResourcesRequired] | None: ...
 
 
 @runtime_checkable

reconcile/openshift_cluster_bots.py

@@ -301,7 +301,7 @@ def filter_clusters(clusters: list[ClusterV1]) -> list[ClusterV1]:
 
 def get_ocm_map(clusters: list[ClusterV1]) -> OCMMap:
     settings = queries.get_app_interface_settings()
-    clusters_info = [c.dict(by_alias=True) for c in clusters]
+    clusters_info = [c.model_dump(by_alias=True) for c in clusters]
     return OCMMap(
         settings=settings,
         clusters=clusters_info,

reconcile/openshift_namespace_labels.py

@@ -229,7 +229,7 @@ def get_gql_namespaces_in_shard() -> list[NamespaceV1]:
     return [
         ns
         for ns in all_namespaces
-        if not ob.is_namespace_deleted(ns.dict(by_alias=True))
+        if not ob.is_namespace_deleted(ns.model_dump(by_alias=True))
         and is_in_shard(f"{ns.cluster.name}/{ns.name}")
     ]