qontract-reconcile 0.10.2.dev361__py3-none-any.whl → 0.10.2.dev474__py3-none-any.whl

reconcile/ocm_machine_pools.py

@@ -7,11 +7,7 @@ from collections.abc import Iterable, Mapping
 from enum import Enum
 from typing import Any, Self
 
-from pydantic import (
-    BaseModel,
-    Field,
-    root_validator,
-)
+from pydantic import BaseModel, Field, SerializeAsAny, model_validator
 
 from reconcile import queries
 from reconcile.gql_definitions.common.clusters import (
@@ -22,6 +18,7 @@ from reconcile.gql_definitions.common.clusters import (
 from reconcile.typed_queries.clusters import get_clusters
 from reconcile.utils.differ import diff_mappings
 from reconcile.utils.disabled_integrations import integration_is_enabled
+from reconcile.utils.json import json_dumps
 from reconcile.utils.ocm import (
     DEFAULT_OCM_MACHINE_POOL_ID,
     OCM,
@@ -61,7 +58,8 @@ class MachinePoolAutoscaling(AbstractAutoscaling):
     min_replicas: int
     max_replicas: int
 
-    @root_validator()
+    @model_validator(mode="before")
+    @classmethod
     def max_greater_min(cls, field_values: Mapping[str, Any]) -> Mapping[str, Any]:
         min_replicas = field_values.get("min_replicas")
         max_replicas = field_values.get("max_replicas")
@@ -82,7 +80,8 @@ class NodePoolAutoscaling(AbstractAutoscaling):
     min_replica: int
     max_replica: int
 
-    @root_validator()
+    @model_validator(mode="before")
+    @classmethod
     def max_greater_min(cls, field_values: Mapping[str, Any]) -> Mapping[str, Any]:
         min_replica = field_values.get("min_replica")
         max_replica = field_values.get("max_replica")
@@ -103,14 +102,15 @@ class AbstractPool(ABC, BaseModel):
     # Abstract class for machine pools, to be implemented by OSD/HyperShift classes
 
     id: str
-    replicas: int | None
-    taints: list[Mapping[str, str]] | None
-    labels: Mapping[str, str] | None
+    replicas: int | None = None
+    taints: list[Mapping[str, str]] | None = None
+    labels: Mapping[str, str] | None = None
     cluster: str
     cluster_type: ClusterType = Field(..., exclude=True)
-    autoscaling: AbstractAutoscaling | None
+    autoscaling: SerializeAsAny[AbstractAutoscaling] | None = None
 
-    @root_validator()
+    @model_validator(mode="before")
+    @classmethod
     def validate_scaling(cls, field_values: Mapping[str, Any]) -> Mapping[str, Any]:
         if field_values.get("autoscaling") and field_values.get("replicas"):
             raise ValueError("autoscaling and replicas are mutually exclusive")
@@ -154,13 +154,13 @@ class MachinePool(AbstractPool):
     instance_type: str
 
     def delete(self, ocm: OCM) -> None:
-        ocm.delete_machine_pool(self.cluster, self.dict(by_alias=True))
+        ocm.delete_machine_pool(self.cluster, self.model_dump(by_alias=True))
 
     def create(self, ocm: OCM) -> None:
-        ocm.create_machine_pool(self.cluster, self.dict(by_alias=True))
+        ocm.create_machine_pool(self.cluster, self.model_dump(by_alias=True))
 
     def update(self, ocm: OCM) -> None:
-        update_dict = self.dict(by_alias=True)
+        update_dict = self.model_dump(by_alias=True)
         # can not update instance_type
         del update_dict["instance_type"]
         if not update_dict["labels"]:
@@ -170,7 +170,10 @@ class MachinePool(AbstractPool):
         ocm.update_machine_pool(self.cluster, update_dict)
 
     def has_diff(self, pool: ClusterMachinePoolV1) -> bool:
-        if self.taints != pool.taints or self.labels != pool.labels:
+        pool_taints = (
+            [p.model_dump(by_alias=True) for p in pool.taints] if pool.taints else None
+        )
+        if self.taints != pool_taints or self.labels != pool.labels:
             logging.warning(
                 f"updating labels or taints for machine pool {pool.q_id} "
                 f"will only be applied to new Nodes"
@@ -178,7 +181,7 @@ class MachinePool(AbstractPool):
 
         return (
             self.replicas != pool.replicas
-            or self.taints != pool.taints
+            or self.taints != pool_taints
             or self.labels != pool.labels
             or self.instance_type != pool.instance_type
             or self._has_diff_autoscale(pool)
@@ -214,7 +217,7 @@ class MachinePool(AbstractPool):
             replicas=pool.replicas,
             autoscaling=autoscaling,
             instance_type=pool.instance_type,
-            taints=[p.dict(by_alias=True) for p in pool.taints or []],
+            taints=[p.model_dump(by_alias=True) for p in pool.taints or []],
             labels=pool.labels,
             cluster=cluster,
             cluster_type=cluster_type,
@@ -232,14 +235,14 @@ class NodePool(AbstractPool):
     subnet: str | None
 
     def delete(self, ocm: OCM) -> None:
-        ocm.delete_node_pool(self.cluster, self.dict(by_alias=True))
+        ocm.delete_node_pool(self.cluster, self.model_dump(by_alias=True))
 
     def create(self, ocm: OCM) -> None:
-        spec = self.dict(by_alias=True)
+        spec = self.model_dump(by_alias=True)
         ocm.create_node_pool(self.cluster, spec)
 
     def update(self, ocm: OCM) -> None:
-        update_dict = self.dict(by_alias=True)
+        update_dict = self.model_dump(by_alias=True)
         # can not update instance_type
         del update_dict["aws_node_pool"]
         # can not update subnet
@@ -251,7 +254,10 @@ class NodePool(AbstractPool):
         ocm.update_node_pool(self.cluster, update_dict)
 
     def has_diff(self, pool: ClusterMachinePoolV1) -> bool:
-        if self.taints != pool.taints or self.labels != pool.labels:
+        pool_taints = (
+            [p.model_dump(by_alias=True) for p in pool.taints] if pool.taints else None
+        )
+        if self.taints != pool_taints or self.labels != pool.labels:
             logging.warning(
                 f"updating labels or taints for node pool {pool.q_id} "
                 f"will only be applied to new Nodes"
@@ -259,7 +265,7 @@ class NodePool(AbstractPool):
 
         return (
             self.replicas != pool.replicas
-            or self.taints != pool.taints
+            or self.taints != pool_taints
             or self.labels != pool.labels
             or self.aws_node_pool.instance_type != pool.instance_type
             or self.subnet != pool.subnet
@@ -297,7 +303,7 @@ class NodePool(AbstractPool):
             aws_node_pool=AWSNodePool(
                 instance_type=pool.instance_type,
             ),
-            taints=[p.dict(by_alias=True) for p in pool.taints or []],
+            taints=[p.model_dump(by_alias=True) for p in pool.taints or []],
             labels=pool.labels,
             subnet=pool.subnet,
             cluster=cluster,
@@ -312,7 +318,7 @@ class PoolHandler(BaseModel):
     pool: AbstractPool
 
     def act(self, dry_run: bool, ocm: OCM) -> None:
-        logging.info(f"{self.action} {self.pool.dict(by_alias=True)}")
+        logging.info(f"{self.action} {self.pool.model_dump(by_alias=True)}")
         if dry_run:
             return
 
@@ -469,7 +475,7 @@ def calculate_diff(
         if invalid_diff:
             errors.append(
                 InvalidUpdateError(
-                    f"can not update {invalid_diff} for existing machine pool on cluster {cluster_name}, CURRENT: {diff_pair.current.json()}, DESIRED: {diff_pair.desired.json()}"
+                    f"can not update {invalid_diff} for existing machine pool on cluster {cluster_name}, CURRENT: {json_dumps(diff_pair.current)}, DESIRED: {json_dumps(diff_pair.desired)}"
                 )
             )
         else:
@@ -531,7 +537,7 @@ def run(dry_run: bool) -> None:
 
     settings = queries.get_app_interface_settings()
     cluster_like_objects = [
-        cluster.dict(by_alias=True) for cluster in filtered_clusters
+        cluster.model_dump(by_alias=True) for cluster in filtered_clusters
     ]
     ocm_map = OCMMap(
         clusters=cluster_like_objects,

reconcile/openshift_base.py

@@ -29,10 +29,14 @@ from reconcile.utils import (
     metrics,
 )
 from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
+from reconcile.utils.differ import DiffPair
 from reconcile.utils.oc import (
+    POD_RECYCLE_SUPPORTED_OWNER_KINDS,
+    AmbiguousResourceTypeError,
     DeploymentFieldIsImmutableError,
     FieldIsImmutableError,
     InvalidValueApplyError,
+    KindNotFoundError,
     MayNotChangeOnceSetError,
     MetaDataAnnotationsTooLongApplyError,
     OC_Map,
@@ -60,6 +64,10 @@ AUTH_METHOD_USER_KEY = {
     "oidc": "org_username",
     "rhidp": "org_username",
 }
+RECYCLE_POD_ANNOTATIONS = [
+    "kubectl.kubernetes.io/restartedAt",
+    "openshift.openshift.io/restartedAt",
+]
 
 
 class ValidationError(Exception):
@@ -128,6 +136,29 @@ class ClusterMap(Protocol):
     ) -> list[str]: ...
 
 
+def validate_managed_resource_types(
+    oc: OCCli,
+    managed_resource_types: Iterable[str],
+    managed_resource_names: Iterable[Mapping[str, Any]],
+    cluster_scope_resource_validation: bool,
+) -> None:
+    """Validate the managed resource types."""
+    managed_resources = [
+        managed_resource_name["resource"]
+        for managed_resource_name in managed_resource_names
+    ]
+    for managed_resource_type in managed_resource_types:
+        # The k8s kind must be supported by the cluster
+        resource = oc.get_api_resource(managed_resource_type)
+
+        if cluster_scope_resource_validation and not resource.namespaced:
+            # cluster-scoped resources must be use managedResourceNames!
+            if managed_resource_type not in managed_resources:
+                raise ValidationError(
+                    f"Cluster-scoped resource {managed_resource_type} must be managed by name only. Please use 'managedResourceNames' field to specify the names of the resources to manage."
+                )
+
+
 def init_specs_to_fetch(
     ri: ResourceInventory,
     oc_map: ClusterMap,
@@ -136,6 +167,7 @@ def init_specs_to_fetch(
     override_managed_types: Iterable[str] | None = None,
     managed_types_key: str = "managedResourceTypes",
     cluster_admin: bool = False,
+    cluster_scope_resource_validation: bool = False,
 ) -> list[StateSpec]:
     state_specs: list[StateSpec] = []
 
@@ -163,9 +195,27 @@ def init_specs_to_fetch(
                 logging.log(level=ex.log_level, msg=ex.message)
                 continue
 
+            managed_resource_names = namespace_info.get("managedResourceNames") or []
+            try:
+                validate_managed_resource_types(
+                    oc,
+                    managed_types,
+                    managed_resource_names,
+                    cluster_scope_resource_validation=cluster_scope_resource_validation,
+                )
+            except KindNotFoundError:
+                # We must allow kinds that are not supported by the cluster because:
+                # 1. We install CRD with an operator in the same MR
+                # 2. SAAS files initialize the namespace objects with managedResourceTypes from the SAAS file
+                #    and we can't expect that all of those are valid for all clusters
+                pass
+            except (AmbiguousResourceTypeError, ValidationError) as e:
+                ri.register_error()
+                logging.error(f"[{cluster}/{namespace_info['name']}] {e}")
+                continue
+
             namespace = namespace_info["name"]
             # These may exit but have a value of None
-            managed_resource_names = namespace_info.get("managedResourceNames") or []
             managed_resource_type_overrides = (
                 namespace_info.get("managedResourceTypeOverrides") or []
             )
@@ -340,6 +390,7 @@ def fetch_current_state(
     cluster_admin: bool = False,
     caller: str | None = None,
     init_projects: bool = False,
+    cluster_scope_resource_validation: bool = False,
 ) -> tuple[ResourceInventory, OC_Map]:
     ri = ResourceInventory()
     settings = queries.get_app_interface_settings()
@@ -362,6 +413,7 @@ def fetch_current_state(
         clusters=clusters,
         override_managed_types=override_managed_types,
         cluster_admin=cluster_admin,
+        cluster_scope_resource_validation=cluster_scope_resource_validation,
     )
     threaded.run(
         populate_current_state,
@@ -542,7 +594,7 @@ def apply(
                 oc.resize_pvcs(namespace, owned_pvc_names, desired_storage)
 
     if recycle_pods:
-        oc.recycle_pods(dry_run, namespace, resource_type, resource)
+        oc.recycle_pods(dry_run, namespace, resource)
 
 
 def create(
@@ -786,10 +838,56 @@ def handle_identical_resources(
     return actions
 
 
+def patch_desired_resource_for_recycle_annotations(
+    desired: OR,
+    current: OR,
+) -> OR:
+    """
+    Patch desired resource with recycle annotations to pod template from current resource.
+    This is to avoid full pods recycle when changes are not affecting pod template.
+    Note desired annotations can override current annotations.
+    For example, if desired resource has kubectl.kubernetes.io/restartedAt defined,
+    it will be used instead of current resource annotation.
+
+    Args:
+        desired: desired resource
+        current: current resource
+
+    Returns:
+        patched desired resource
+    """
+    if current.kind not in POD_RECYCLE_SUPPORTED_OWNER_KINDS:
+        return desired
+
+    current_annotations = (
+        current.body.get("spec", {})
+        .get("template", {})
+        .get("metadata", {})
+        .get("annotations")
+        or {}
+    )
+    patch_annotations = {
+        k: value
+        for k in RECYCLE_POD_ANNOTATIONS
+        if (value := current_annotations.get(k))
+    }
+    if patch_annotations:
+        desired_annotations = (
+            desired.body.setdefault("spec", {})
+            .setdefault("template", {})
+            .setdefault("metadata", {})
+            .setdefault("annotations", {})
+        )
+        desired.body["spec"]["template"]["metadata"]["annotations"] = (
+            patch_annotations | desired_annotations
+        )
+    return desired
+
+
 def handle_modified_resources(
     oc_map: ClusterMap,
     ri: ResourceInventory,
-    modified_resources: Mapping[Any, Any],
+    modified_resources: Mapping[str, DiffPair[OR, OR]],
     cluster: str,
     namespace: str,
     resource_type: str,
@@ -985,6 +1083,12 @@ def _realize_resource_data_3way_diff(
     if options.enable_deletion and options.override_enable_deletion is False:
         options.enable_deletion = False
 
+    for k in data["current"].keys() & data["desired"].keys():
+        patch_desired_resource_for_recycle_annotations(
+            desired=data["desired"][k],
+            current=data["current"][k],
+        )
+
     diff_result = differ.diff_mappings(
         data["current"], data["desired"], equal=three_way_diff_using_hash
     )
@@ -1363,6 +1467,11 @@ class HasOpenShiftResources(Protocol):
     openshift_resources: list | None
 
 
+@runtime_checkable
+class HasOpenShiftResourcesRequired(Protocol):
+    openshift_resources: list
+
+
 @runtime_checkable
 class HasOpenshiftServiceAccountTokens(Protocol):
     openshift_service_account_tokens: list | None
@@ -1371,7 +1480,7 @@ class HasOpenshiftServiceAccountTokens(Protocol):
 @runtime_checkable
 class HasSharedResourcesOpenShiftResources(Protocol):
     @property
-    def shared_resources(self) -> Sequence[HasOpenShiftResources] | None: ...
+    def shared_resources(self) -> Sequence[HasOpenShiftResourcesRequired] | None: ...
 
 
 @runtime_checkable

reconcile/openshift_cluster_bots.py

@@ -301,7 +301,7 @@ def filter_clusters(clusters: list[ClusterV1]) -> list[ClusterV1]:
 
 def get_ocm_map(clusters: list[ClusterV1]) -> OCMMap:
     settings = queries.get_app_interface_settings()
-    clusters_info = [c.dict(by_alias=True) for c in clusters]
+    clusters_info = [c.model_dump(by_alias=True) for c in clusters]
     return OCMMap(
         settings=settings,
         clusters=clusters_info,

reconcile/openshift_namespace_labels.py

@@ -229,7 +229,7 @@ def get_gql_namespaces_in_shard() -> list[NamespaceV1]:
     return [
         ns
         for ns in all_namespaces
-        if not ob.is_namespace_deleted(ns.dict(by_alias=True))
+        if not ob.is_namespace_deleted(ns.model_dump(by_alias=True))
         and is_in_shard(f"{ns.cluster.name}/{ns.name}")
     ]
 

reconcile/openshift_namespaces.py

@@ -1,25 +1,27 @@
 import logging
-import sys
+from collections import defaultdict
 from collections.abc import (
     Callable,
     Iterable,
-    Mapping,
     Sequence,
 )
+from dataclasses import dataclass
+from enum import StrEnum
 from typing import Any
 
 from sretoolbox.utils import threaded
 
 import reconcile.openshift_base as ob
 from reconcile.gql_definitions.common.namespaces_minimal import NamespaceV1
-from reconcile.status import ExitCodes
 from reconcile.typed_queries.app_interface_vault_settings import (
     get_app_interface_vault_settings,
 )
 from reconcile.typed_queries.namespaces_minimal import get_namespaces_minimal
 from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
 from reconcile.utils.defer import defer
-from reconcile.utils.oc_filters import filter_namespaces_by_cluster_and_namespace
+from reconcile.utils.oc_filters import (
+    filter_namespaces_by_cluster_and_namespace,
+)
 from reconcile.utils.oc_map import (
     OCLogMsg,
     OCMap,
@@ -30,113 +32,111 @@ from reconcile.utils.sharding import is_in_shard
 
 QONTRACT_INTEGRATION = "openshift-namespaces"
 
-NS_STATE_PRESENT = "present"
-NS_STATE_ABSENT = "absent"
-
-NS_ACTION_CREATE = "create"
-NS_ACTION_DELETE = "delete"
 
+class Action(StrEnum):
+    CREATE = "create"
+    DELETE = "delete"
 
-DUPLICATES_LOG_MSG = "Found multiple definitions for the namespace {key}"
 
+@dataclass(frozen=True)
+class DesiredState:
+    cluster: str
+    namespace: str
+    delete: bool
+    cluster_admin: bool
 
-def get_desired_state(namespaces: Iterable[NamespaceV1]) -> list[dict[str, str]]:
-    desired_state: list[dict[str, str]] = []
-    for ns in namespaces:
-        state = NS_STATE_PRESENT
-        if ns.delete:
-            state = NS_STATE_ABSENT
 
-        desired_state.append({
-            "cluster": ns.cluster.name,
-            "namespace": ns.name,
-            "desired_state": state,
-        })
+class NamespaceDuplicateError(Exception):
+    pass
 
-    return desired_state
 
+def get_namespaces(
+    cluster_name: Sequence[str] | None,
+    namespace_name: Sequence[str] | None,
+) -> tuple[list[NamespaceV1], list[NamespaceDuplicateError]]:
+    all_namespaces = get_namespaces_minimal()
 
-def get_shard_namespaces(
-    namespaces: Iterable[NamespaceV1],
-) -> tuple[list[NamespaceV1], bool]:
-    # Structure holding duplicates by namespace key
-    duplicates: dict[str, list[NamespaceV1]] = {}
-    # namespace filtered list without duplicates
-    filtered_ns: dict[str, NamespaceV1] = {}
-
-    err = False
-    for ns in namespaces:
-        key = f"{ns.cluster.name}/{ns.name}"
+    namespaces_by_shard_key = defaultdict(list)
 
+    for namespace in all_namespaces:
+        key = f"{namespace.cluster.name}/{namespace.name}"
         if is_in_shard(key):
-            if key not in filtered_ns:
-                filtered_ns[key] = ns
-            else:
-                # Duplicated NS
-                dupe_list_by_key = duplicates.setdefault(key, [])
-                dupe_list_by_key.append(ns)
-
-    for key, dupe_list in duplicates.items():
-        dupe_list.append(filtered_ns[key])
-        delete_flags = (
-            [ns.delete for ns in dupe_list_by_key] if dupe_list_by_key else []
-        )
+            namespaces_by_shard_key[key].append(namespace)
 
-        if len(set(delete_flags)) > 1:
-            # If true only some definitions in list have the delete flag.
-            # this case will generate an error
-            err = True
-            # Remove the namespace found from the filtered list
-            del filtered_ns[key]
-            logging.error(DUPLICATES_LOG_MSG.format(key=key))
+    managed_namespaces = []
+    duplicate_errors = []
+
+    for key, namespaces in namespaces_by_shard_key.items():
+        if len(namespaces) == 1:
+            namespace = namespaces[0]
+            if not namespace.managed_by_external:
+                managed_namespaces.append(namespace)
         else:
-            # If all namespaces have the same delete option
-            # The action will be performaed
-            logging.debug(DUPLICATES_LOG_MSG.format(key=key))
+            msg = f"Found multiple definitions for the namespace {key}"
+            duplicate_errors.append(NamespaceDuplicateError(msg))
+            logging.error(msg)
 
-    return list(filtered_ns.values()), err
+    namespaces = filter_namespaces_by_cluster_and_namespace(
+        namespaces=managed_namespaces,
+        cluster_names=cluster_name,
+        namespace_names=namespace_name,
+    )
 
+    return namespaces, duplicate_errors
 
-def manage_namespaces(spec: Mapping[str, str], oc_map: OCMap, dry_run: bool) -> None:
-    cluster = spec["cluster"]
-    namespace = spec["namespace"]
-    desired_state = spec["desired_state"]
 
-    oc = oc_map.get(cluster)
+def build_desired_state(
+    namespaces: Iterable[NamespaceV1],
+) -> list[DesiredState]:
+    return [
+        DesiredState(
+            cluster=namespace.cluster.name,
+            namespace=namespace.name,
+            delete=namespace.delete or False,
+            cluster_admin=namespace.cluster_admin or False,
+        )
+        for namespace in namespaces
+    ]
+
+
+def manage_namespace(
+    desired_state: DesiredState,
+    oc_map: OCMap,
+    dry_run: bool,
+) -> None:
+    namespace = desired_state.namespace
+
+    oc = oc_map.get(desired_state.cluster, privileged=desired_state.cluster_admin)
     if isinstance(oc, OCLogMsg):
         logging.log(level=oc.log_level, msg=oc.message)
-        return None
+        return
 
-    act = {NS_ACTION_CREATE: oc.new_project, NS_ACTION_DELETE: oc.delete_project}
+    current_delete = not oc.project_exists(namespace)
 
-    exists = oc.project_exists(namespace)
-    action = None
-    if not exists and desired_state == NS_STATE_PRESENT:
-        if namespace.startswith("openshift-"):
-            raise ValueError('cannot request a project starting with "openshift-"')
-        action = NS_ACTION_CREATE
-    elif exists and desired_state == NS_STATE_ABSENT:
-        action = NS_ACTION_DELETE
+    if desired_state.delete == current_delete:
+        return
 
-    if action:
-        logging.info([action, cluster, namespace])
-        if not dry_run:
-            act[action](namespace)
+    action = Action.DELETE if desired_state.delete else Action.CREATE
 
+    logging.info([str(action), desired_state.cluster, namespace])
+    if not dry_run:
+        match action:
+            case Action.CREATE:
+                oc.new_project(namespace)
+            case Action.DELETE:
+                oc.delete_project(namespace)
 
-def check_results(
-    desired_state: Iterable[Mapping[str, str]], results: Iterable[Any]
-) -> bool:
-    err = False
+
+def build_runtime_errors(
+    desired_state: Iterable[DesiredState],
+    results: Iterable[Any],
+) -> list[Exception]:
+    exceptions = []
     for s, e in zip(desired_state, results, strict=False):
         if isinstance(e, Exception):
-            err = True
-            msg = (
-                f"cluster: {s['cluster']}, namespace: {s['namespace']}, "
-                f"exception: {e!s}"
-            )
-            logging.error(msg)
-    return err
+            e.add_note(f"cluster: {s.cluster}, namespace: {s.namespace}")
+            exceptions.append(e)
+    return exceptions
 
 
 @defer
@@ -149,15 +149,8 @@ def run(
     namespace_name: Sequence[str] | None = None,
     defer: Callable | None = None,
 ) -> None:
-    all_namespaces = get_namespaces_minimal()
-    shard_namespaces, duplicates = get_shard_namespaces(all_namespaces)
-    namespaces = filter_namespaces_by_cluster_and_namespace(
-        namespaces=shard_namespaces,
-        cluster_names=cluster_name,
-        namespace_names=namespace_name,
-    )
-
-    desired_state = get_desired_state(namespaces)
+    namespaces, duplicate_errors = get_namespaces(cluster_name, namespace_name)
+    desired_state = build_desired_state(namespaces)
 
     vault_settings = get_app_interface_vault_settings()
     secret_reader = create_secret_reader(use_vault=vault_settings.vault)
@@ -171,16 +164,17 @@ def run(
         thread_pool_size=thread_pool_size,
         init_projects=True,
     )
-
     if defer:
         defer(oc_map.cleanup)
 
     ob.publish_cluster_desired_metrics_from_state(
-        desired_state, QONTRACT_INTEGRATION, "Namespace"
+        state=({"cluster": s.cluster} for s in desired_state),
+        integration=QONTRACT_INTEGRATION,
+        kind="Namespace",
     )
 
     results = threaded.run(
-        manage_namespaces,
+        manage_namespace,
         desired_state,
         thread_pool_size,
         return_exceptions=True,
@@ -188,6 +182,7 @@ def run(
         oc_map=oc_map,
     )
 
-    err = check_results(desired_state=desired_state, results=results)
-    if err or duplicates:
-        sys.exit(ExitCodes.ERROR)
+    runtime_errors = build_runtime_errors(desired_state, results)
+    errors = runtime_errors + duplicate_errors
+    if errors:
+        raise ExceptionGroup("Reconcile errors occurred", errors)