qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/openshift_resources_base.py

@@ -3,37 +3,38 @@ import hashlib
 import itertools
 import json
 import logging
+import re
 import sys
+from collections import defaultdict
 from collections.abc import (
+    Callable,
+    Generator,
     Iterable,
     Mapping,
+    MutableMapping,
+    Sequence,
 )
 from contextlib import contextmanager
 from dataclasses import dataclass
-from functools import cache
 from textwrap import indent
 from threading import Lock
 from typing import (
     Any,
-    Optional,
     Protocol,
-    Tuple,
 )
-from urllib import parse
+from unittest.mock import DEFAULT, patch
 
 import anymarkup
-import jinja2
-import jinja2.sandbox
+from deepdiff import DeepHash
 from sretoolbox.utils import (
-    retry,
     threaded,
 )
 
 import reconcile.openshift_base as ob
+import reconcile.utils.jinja2.utils as jinja2_utils
 from reconcile import queries
 from reconcile.change_owners.diff import IDENTIFIER_FIELD_NAME
-from reconcile.checkpoint import url_makes_sense
-from reconcile.github_users import init_github
+from reconcile.external_resources.meta import SECRET_UPDATED_AT
 from reconcile.utils import (
     amtool,
     gql,
@@ -41,9 +42,10 @@ from reconcile.utils import (
 )
 from reconcile.utils.defer import defer
 from reconcile.utils.exceptions import FetchResourceError
-from reconcile.utils.jinja2_ext import (
-    B64EncodeExtension,
-    RaiseErrorExtension,
+from reconcile.utils.jinja2.utils import (
+    FetchSecretError,
+    process_extracurlyjinja2_template,
+    process_jinja2_template,
 )
 from reconcile.utils.oc import (
     OC_Map,
@@ -51,12 +53,14 @@ from reconcile.utils.oc import (
     OCLogMsg,
     StatusCodeError,
 )
-from reconcile.utils.openshift_resource import ConstructResourceError
-from reconcile.utils.openshift_resource import OpenshiftResource as OR
 from reconcile.utils.openshift_resource import (
+    ConstructResourceError,
     ResourceInventory,
     ResourceKeyExistsError,
+    ResourceNotManagedError,
+    base64_encode_secret_field_value,
 )
+from reconcile.utils.openshift_resource import OpenshiftResource as OR
 from reconcile.utils.runtime.integration import DesiredStateShardConfig
 from reconcile.utils.secret_reader import SecretReader
 from reconcile.utils.semver_helper import make_semver
@@ -142,6 +146,8 @@ NAMESPACES_QUERY = """
 {
   namespaces: namespaces_v1 {
     name
+    path
+    labels
     delete
     clusterAdmin
     managedResourceTypes
@@ -161,8 +167,13 @@ NAMESPACES_QUERY = """
     openshiftResources {
       %s
     }
+    environment {
+      name
+      parameters
+    }
     cluster {
       name
+      labels
       serverUrl
         auth {
           service
@@ -181,8 +192,14 @@ NAMESPACES_QUERY = """
         %s
       }
       spec {
+        ... on ClusterSpecROSA_v1 {
+          account {
+            uid
+          }
+        }
         version
         region
+        hypershift
       }
       network {
         pod
@@ -215,203 +232,58 @@ NAMESPACES_QUERY = """
 QONTRACT_INTEGRATION = "openshift_resources_base"
 QONTRACT_INTEGRATION_VERSION = make_semver(1, 9, 2)
 QONTRACT_BASE64_SUFFIX = "_qb64"
-APP_INT_BASE_URL = "https://gitlab.cee.redhat.com/service/app-interface"
+KUBERNETES_SECRET_DATA_KEY_RE = "^[-._a-zA-Z0-9]+$"
 
+# Keys in vault secrets that do not need to land
+# into K8S secrets.
+VAULT_SECRETS_EXCLUDED_KEYS = {SECRET_UPDATED_AT}
 _log_lock = Lock()
 
 
-def _locked_info_log(msg: str):
+def _locked_info_log(msg: str) -> None:
     with _log_lock:
         logging.info(msg)
 
 
-def _locked_debug_log(msg: str):
+def _locked_debug_log(msg: str) -> None:
     with _log_lock:
         logging.debug(msg)
 
 
-def _locked_error_log(msg: str):
+def _locked_error_log(msg: str) -> None:
     with _log_lock:
         logging.error(msg)
 
 
-class FetchSecretError(Exception):
-    def __init__(self, msg):
-        super().__init__("error fetching secret: " + str(msg))
-
-
 class FetchRouteError(Exception):
-    def __init__(self, msg):
+    def __init__(self, msg: Any):
         super().__init__("error fetching route: " + str(msg))
 
 
-class Jinja2TemplateError(Exception):
-    def __init__(self, msg):
-        super().__init__("error processing jinja2 template: " + str(msg))
+class ResourceTemplateRenderError(Exception):
+    pass
 
 
-class ResourceTemplateRenderError(Exception):
+class SecretKeyFormatError(Exception):
     pass
 
 
 class UnknownProviderError(Exception):
-    def __init__(self, msg):
+    def __init__(self, msg: Any):
         super().__init__("unknown provider error: " + str(msg))
 
 
 class UnknownTemplateTypeError(Exception):
-    def __init__(self, msg):
+    def __init__(self, msg: Any):
         super().__init__("unknown template type error: " + str(msg))
 
 
-@retry()
-def lookup_secret(path, key, version=None, tvars=None, settings=None):
-    if tvars is not None:
-        path = process_jinja2_template(body=path, vars=tvars, settings=settings)
-        key = process_jinja2_template(body=key, vars=tvars, settings=settings)
-        if version and not isinstance(version, int):
-            version = process_jinja2_template(
-                body=version, vars=tvars, settings=settings
-            )
-    secret = {"path": path, "field": key, "version": version}
-    try:
-        secret_reader = SecretReader(settings)
-        return secret_reader.read(secret)
-    except Exception as e:
-        raise FetchSecretError(e)
-
-
-def lookup_github_file_content(repo, path, ref, tvars=None, settings=None):
-    if tvars is not None:
-        repo = process_jinja2_template(body=repo, vars=tvars, settings=settings)
-        path = process_jinja2_template(body=path, vars=tvars, settings=settings)
-        ref = process_jinja2_template(body=ref, vars=tvars, settings=settings)
-
-    gh = init_github()
-    c = gh.get_repo(repo).get_contents(path, ref).decoded_content
-    return c.decode("utf-8")
-
-
-def lookup_graphql_query_results(query: str, **kwargs) -> list[Any]:
-    gqlapi = gql.get_api()
-    resource = gqlapi.get_resource(query)["content"]
-    rendered_resource = jinja2.Template(resource).render(**kwargs)
-    results = list(gqlapi.query(rendered_resource).values())[0]
-    return results
-
-
-def json_to_dict(input):
-    """Jinja2 filter to parse JSON strings into dictionaries.
-       This becomes useful to access Graphql queries data (labels)
-    :param input: json string
-    :return: dict with the parsed inputs contents
-    """
-    data = json.loads(input)
-    return data
-
-
-def urlescape(
-    string: str,
-    safe: str = "/",
-    encoding: Optional[str] = None,
-) -> str:
-    """Jinja2 filter that is a simple wrapper around urllib's URL quoting
-    functions that takes a string value and makes it safe for use as URL
-    components escaping any reserved characters using URL encoding. See:
-    urllib.parse.quote() and urllib.parse.quote_plus() for reference.
-
-    :param str string: String value to escape.
-    :param str safe: Optional characters that should not be escaped.
-    :param encoding: Encoding to apply to the string to be escaped. Defaults
-        to UTF-8. Unsupported characters raise a UnicodeEncodeError error.
-    :type encoding: typing.Optional[str]
-    :returns: A string with reserved characters escaped.
-    :rtype: str
-    """
-    return parse.quote(string, safe=safe, encoding=encoding)
-
-
-def urlunescape(string: str, encoding: Optional[str] = None) -> str:
-    """Jinja2 filter that is a simple wrapper around urllib's URL unquoting
-    functions that takes an URL-encoded string value and unescapes it
-    replacing any URL-encoded values with their character equivalent. See:
-    urllib.parse.unquote() and urllib.parse.unquote_plus() for reference.
-
-    :param str string: String value to unescape.
-    :param encoding: Encoding to apply to the string to be unescaped. Defaults
-        to UTF-8. Unsupported characters are replaced by placeholder values.
-    :type encoding: typing.Optional[str]
-    :returns: A string with URL-encoded sequences unescaped.
-    :rtype: str
-    """
-    if encoding is None:
-        encoding = "utf-8"
-    return parse.unquote(string, encoding=encoding)
-
-
-@cache
-def compile_jinja2_template(body, extra_curly: bool = False):
-    env: dict = {}
-    if extra_curly:
-        env = {
-            "block_start_string": "{{%",
-            "block_end_string": "%}}",
-            "variable_start_string": "{{{",
-            "variable_end_string": "}}}",
-            "comment_start_string": "{{#",
-            "comment_end_string": "#}}",
-        }
-
-    jinja_env = jinja2.sandbox.SandboxedEnvironment(
-        extensions=[B64EncodeExtension, RaiseErrorExtension],
-        undefined=jinja2.StrictUndefined,
-        **env,
-    )
-    jinja_env.filters.update(
-        {
-            "json_to_dict": json_to_dict,
-            "urlescape": urlescape,
-            "urlunescape": urlunescape,
-        }
-    )
-
-    return jinja_env.from_string(body)
-
-
-def process_jinja2_template(body, vars=None, extra_curly: bool = False, settings=None):
-    if vars is None:
-        vars = {}
-    vars.update(
-        {
-            "vault": lambda p, k, v=None: lookup_secret(
-                path=p, key=k, version=v, tvars=vars, settings=settings
-            ),
-            "github": lambda u, p, r, v=None: lookup_github_file_content(
-                repo=u, path=p, ref=r, tvars=vars, settings=settings
-            ),
-            "urlescape": lambda u, s="/", e=None: urlescape(
-                string=u, safe=s, encoding=e
-            ),
-            "urlunescape": lambda u, e=None: urlunescape(string=u, encoding=e),
-            "query": lookup_graphql_query_results,
-            "url": url_makes_sense,
-        }
-    )
-    try:
-        template = compile_jinja2_template(body, extra_curly)
-        r = template.render(vars)
-    except Exception as e:
-        raise Jinja2TemplateError(e)
-    return r
-
-
-def process_extracurlyjinja2_template(body, vars=None, env=None, settings=None):
-    if vars is None:
-        vars = {}
-    return process_jinja2_template(body, vars=vars, extra_curly=True, settings=settings)
-
-
-def check_alertmanager_config(data, path, alertmanager_config_key, decode_base64=False):
+def check_alertmanager_config(
+    data: Mapping[str, Any],
+    path: str,
+    alertmanager_config_key: str,
+    decode_base64: bool = False,
+) -> None:
     try:
         config = data[alertmanager_config_key]
     except KeyError:
@@ -419,7 +291,7 @@ def check_alertmanager_config(data, path, alertmanager_config_key, decode_base64
             f"error validating alertmanager config in {path}: "
             f"missing key {alertmanager_config_key}"
         )
-        raise FetchResourceError(e_msg)
+        raise FetchResourceError(e_msg) from None
 
     if decode_base64:
         config = base64.b64decode(config).decode("utf-8")
@@ -431,15 +303,15 @@ def check_alertmanager_config(data, path, alertmanager_config_key, decode_base64
 
 
 def fetch_provider_resource(
-    resource: dict,
-    tfunc=None,
-    tvars=None,
-    validate_json=False,
-    validate_alertmanager_config=False,
-    alertmanager_config_key="alertmanager.yaml",
-    add_path_to_prom_rules=True,
-    skip_validation=False,
-    settings=None,
+    resource: Mapping,
+    tfunc: Callable | None = None,
+    tvars: Mapping[str, Any] | None = None,
+    validate_json: bool = False,
+    validate_alertmanager_config: bool = False,
+    alertmanager_config_key: str = "alertmanager.yaml",
+    add_path_to_prom_rules: bool = True,
+    skip_validation: bool = False,
+    settings: Mapping[str, Any] | None = None,
 ) -> OR:
     path = resource["path"]
     content = resource["content"]
@@ -461,10 +333,10 @@ def fetch_provider_resource(
     except TypeError:
         body_type = type(body).__name__
         e_msg = f"invalid resource type {body_type} found in path: {path}"
-        raise FetchResourceError(e_msg)
+        raise FetchResourceError(e_msg) from None
     except anymarkup.AnyMarkupError:
         e_msg = f"Could not parse data. Skipping resource: {path}"
-        raise FetchResourceError(e_msg)
+        raise FetchResourceError(e_msg) from None
 
     if validate_json:
         files = body["data"]
@@ -473,7 +345,7 @@ def fetch_provider_resource(
                 json.loads(file_content)
             except ValueError:
                 e_msg = f"invalid json in {path} under {file_name}"
-                raise FetchResourceError(e_msg)
+                raise FetchResourceError(e_msg) from None
 
     if validate_alertmanager_config:
         if body["kind"] == "Secret":
@@ -491,6 +363,9 @@ def fetch_provider_resource(
 
     if add_path_to_prom_rules:
         if body["kind"] == "PrometheusRule":
+            app_int_base_url = "https://gitlab.cee.redhat.com/service/app-interface"
+            if settings and "repoUrl" in settings:
+                app_int_base_url = settings["repoUrl"]
             try:
                 groups = body["spec"]["groups"]
                 for group in groups:
@@ -499,10 +374,9 @@ def fetch_provider_resource(
                         annotations = rule.get("annotations")
                         if not annotations:
                             continue
-                        # TODO(mafriedm): make this better
-                        rule["annotations"][
-                            "html_url"
-                        ] = f"{APP_INT_BASE_URL}/blob/master/resources{path}"
+                        rule["annotations"]["html_url"] = (
+                            f"{app_int_base_url}/blob/master/resources{path}"
+                        )
             except Exception:
                 logging.warning(
                     "could not add html_url annotation to" + body["metadata"]["name"]
@@ -513,31 +387,35 @@ def fetch_provider_resource(
             body, QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION, error_details=path
         )
     except ConstructResourceError as e:
-        raise FetchResourceError(str(e))
+        raise FetchResourceError(str(e)) from None
 
 
 def fetch_provider_vault_secret(
-    path,
-    version,
-    name,
-    labels,
-    annotations,
-    type,
-    integration,
-    integration_version,
-    validate_alertmanager_config=False,
-    alertmanager_config_key="alertmanager.yaml",
-    settings=None,
+    path: str,
+    version: str,
+    name: str,
+    labels: Mapping[str, str] | None,
+    annotations: Mapping[str, str],
+    type: str,
+    integration: str,
+    integration_version: str,
+    validate_alertmanager_config: bool = False,
+    alertmanager_config_key: str = "alertmanager.yaml",
+    settings: Mapping[str, Any] | None = None,
 ) -> OR:
     # get the fields from vault
     secret_reader = SecretReader(settings)
-    raw_data = secret_reader.read_all({"path": path, "version": version})
+    raw_data = {
+        k: v
+        for k, v in secret_reader.read_all({"path": path, "version": version}).items()
+        if k not in VAULT_SECRETS_EXCLUDED_KEYS
+    }
 
     if validate_alertmanager_config:
         check_alertmanager_config(raw_data, path, alertmanager_config_key)
 
     # construct oc resource
-    body = {
+    body: dict[str, Any] = {
         "apiVersion": "v1",
         "kind": "Secret",
         "type": type,
@@ -546,24 +424,42 @@ def fetch_provider_vault_secret(
     if labels:
         body["metadata"]["labels"] = labels
 
+    assert_valid_secret_keys(raw_data)
+
     # populate data
     for k, v in raw_data.items():
-        if v == "":
-            continue
         if k.lower().endswith(QONTRACT_BASE64_SUFFIX):
             k = k[: -len(QONTRACT_BASE64_SUFFIX)]
             v = v.replace("\n", "")
-        elif v is not None:
-            v = base64.b64encode(v.encode()).decode("utf-8")
+        else:
+            v = base64_encode_secret_field_value(v)
         body.setdefault("data", {})[k] = v
 
     try:
         return OR(body, integration, integration_version, error_details=path)
     except ConstructResourceError as e:
-        raise FetchResourceError(str(e))
+        raise FetchResourceError(str(e)) from None
+
+
+# check to ensure that all of the keys are valid by looking to see if there are
+# any white space issues. If any issues are uncovered, an exception will be
+# raised.
+# we're receiving the full key: value information, not simply a list of keys.
+def assert_valid_secret_keys(secrets_data: dict[str, str]) -> None:
+    for k in secrets_data:
+        matches = re.search(KUBERNETES_SECRET_DATA_KEY_RE, k)
+        if not matches:
+            raise SecretKeyFormatError(
+                f"'{k}' is not valid key name for a Secret. a valid Secret key must consist of alphanumeric characters, '-', '_' or '.' (e.g. 'key.name',  or 'KEY_NAME',  or 'key-name', regex used for validation is '^[-._a-zA-Z0-9]+$')"
+            )
 
 
-def fetch_provider_route(resource: dict, tls_path, tls_version, settings=None) -> OR:
+def fetch_provider_route(
+    resource: Mapping,
+    tls_path: str | None,
+    tls_version: str | None,
+    settings: Mapping | None = None,
+) -> OR:
     path = resource["path"]
     openshift_resource = fetch_provider_resource(resource)
 
@@ -589,9 +485,7 @@ def fetch_provider_route(resource: dict, tls_path, tls_version, settings=None) -
             tls[k] = v
             continue
 
-        msg = "Route secret '{}' key '{}' not in valid keys {}".format(
-            tls_path, k, valid_keys
-        )
+        msg = f"Route secret '{tls_path}' key '{k}' not in valid keys {valid_keys}"
         _locked_info_log(msg)
 
     host = openshift_resource.body["spec"].get("host")
@@ -606,12 +500,15 @@ def fetch_provider_route(resource: dict, tls_path, tls_version, settings=None) -
 
 
 def fetch_openshift_resource(
-    resource, parent, settings=None, skip_validation=False
+    resource: Mapping,
+    parent: Mapping[str, Any],
+    settings: Mapping | None = None,
+    skip_validation: bool = False,
 ) -> OR:
     provider = resource["provider"]
     if provider == "resource":
         path = resource["resource"]["path"]
-        _locked_debug_log("Processing {}: {}".format(provider, path))
+        _locked_debug_log(f"Processing {provider}: {path}")
         validate_json = resource.get("validate_json") or False
         add_path_to_prom_rules = resource.get("add_path_to_prom_rules", True)
         validate_alertmanager_config = (
@@ -631,7 +528,7 @@ def fetch_openshift_resource(
         )
     elif provider == "resource-template":
         path = resource["resource"]["path"]
-        _locked_debug_log("Processing {}: {}".format(provider, path))
+        _locked_debug_log(f"Processing {provider}: {path}")
         add_path_to_prom_rules = resource.get("add_path_to_prom_rules", True)
         validate_alertmanager_config = (
             resource.get("validate_alertmanager_config") or False
@@ -664,12 +561,12 @@ def fetch_openshift_resource(
                 settings=settings,
             )
         except Exception as e:
-            msg = "could not render template at path {}\n{}".format(path, e)
-            raise ResourceTemplateRenderError(msg)
+            msg = f"could not render template at path {path}\n{e}"
+            raise ResourceTemplateRenderError(msg) from None
     elif provider == "vault-secret":
         path = resource["path"]
         version = resource["version"]
-        _locked_debug_log("Processing {}: {} - {}".format(provider, path, version))
+        _locked_debug_log(f"Processing {provider}: {path} - {version}")
         rn = resource["name"]
         name = path.split("/")[-1] if rn is None else rn
         rl = resource["labels"]
@@ -699,10 +596,10 @@ def fetch_openshift_resource(
                 settings=settings,
             )
         except (SecretVersionNotFound, SecretVersionIsNone) as e:
-            raise FetchSecretError(e)
+            raise FetchSecretError(e) from None
     elif provider == "route":
         path = resource["resource"]["path"]
-        _locked_debug_log("Processing {}: {}".format(provider, path))
+        _locked_debug_log(f"Processing {provider}: {path}")
         tls_path = resource["vault_tls_secret_path"]
         tls_version = resource["vault_tls_secret_version"]
         openshift_resource = fetch_provider_route(
@@ -710,7 +607,7 @@ def fetch_openshift_resource(
         )
     elif provider == "prometheus-rule":
         path = resource["resource"]["path"]
-        _locked_debug_log("Processing {}: {}".format(provider, path))
+        _locked_debug_log(f"Processing {provider}: {path}")
         add_path_to_prom_rules = resource.get("add_path_to_prom_rules", True)
         tv = {}
         if resource["variables"]:
@@ -738,8 +635,8 @@ def fetch_openshift_resource(
                 settings=settings,
             )
         except Exception as e:
-            msg = "could not render template at path {}\n{}".format(path, e)
-            raise ResourceTemplateRenderError(msg)
+            msg = f"could not render template at path {path}\n{e}"
+            raise ResourceTemplateRenderError(msg) from None
 
     else:
         raise UnknownProviderError(provider)
@@ -753,8 +650,8 @@ def fetch_current_state(
     cluster: str,
     namespace: str,
     kind: str,
-    resource_names=Iterable[str],
-):
+    resource_names: Iterable[str] | None,
+) -> None:
     _locked_debug_log(f"Fetching {kind} from {cluster}/{namespace}")
     if not oc.is_kind_supported(kind):
         logging.warning(f"[{cluster}] cluster has no API resource {kind}.")
@@ -780,8 +677,8 @@ def fetch_desired_state(
     resource: Mapping[str, Any],
     parent: Mapping[str, Any],
     privileged: bool,
-    settings: Optional[Mapping[str, Any]] = None,
-):
+    settings: Mapping[str, Any] | None = None,
+) -> None:
     try:
         openshift_resource = fetch_openshift_resource(resource, parent, settings)
     except (
@@ -791,7 +688,7 @@ def fetch_desired_state(
         UnknownProviderError,
     ) as e:
         ri.register_error()
-        msg = "[{}/{}] {}".format(cluster, namespace, str(e))
+        msg = f"[{cluster}/{namespace}] {e!s}"
         _locked_error_log(msg)
         return
 
@@ -809,9 +706,7 @@ def fetch_desired_state(
         # combination was not initialized, meaning that it shouldn't be
         # managed. But someone is trying to add it via app-interface
         ri.register_error()
-        msg = "[{}/{}] unknown kind: {}. hint: is it missing from managedResourceTypes?".format(
-            cluster, namespace, openshift_resource.kind
-        )
+        msg = f"[{cluster}/{namespace}] unknown kind: {openshift_resource.kind}. hint: is it missing from managedResourceTypes?"
         _locked_error_log(msg)
         return
     except ResourceKeyExistsError:
@@ -819,9 +714,14 @@ def fetch_desired_state(
         # a desired resource with the same name and
         # the same type was already added previously
         ri.register_error()
-        msg = ("[{}/{}] desired item already exists: {}/{}.").format(
-            cluster, namespace, openshift_resource.kind, openshift_resource.name
-        )
+        msg = f"[{cluster}/{namespace}] desired item already exists: {openshift_resource.kind}/{openshift_resource.name}."
+        _locked_error_log(msg)
+        return
+    except ResourceNotManagedError:
+        # This is failing because the resource name is
+        # not in the list of resource names that are managed
+        ri.register_error()
+        msg = f"[{cluster}/{namespace}] desired item is not managed: {openshift_resource.kind}/{openshift_resource.name}."
         _locked_error_log(msg)
         return
 
@@ -829,7 +729,7 @@ def fetch_desired_state(
 def fetch_states(
     spec: ob.StateSpec,
     ri: ResourceInventory,
-    settings: Optional[Mapping[str, Any]] = None,
+    settings: Mapping[str, Any] | None = None,
 ) -> None:
     try:
         if isinstance(spec, ob.CurrentStateSpec):
@@ -855,17 +755,17 @@ def fetch_states(
 
     except StatusCodeError as e:
         ri.register_error(cluster=spec.cluster)
-        logging.error(f"{spec} - exception: {str(e)}")
+        logging.error(f"{spec} - exception: {e!s}")
 
 
 def fetch_data(
-    namespaces,
-    thread_pool_size,
-    internal,
-    use_jump_host,
-    init_api_resources=False,
-    overrides=None,
-):
+    namespaces: Iterable[Mapping[str, Any]],
+    thread_pool_size: int,
+    internal: bool | None,
+    use_jump_host: bool,
+    init_api_resources: bool = False,
+    overrides: Iterable[str] | None = None,
+) -> tuple[OC_Map, ResourceInventory]:
     ri = ResourceInventory()
     settings = queries.get_app_interface_settings()
     logging.debug(f"Overriding keys {overrides}")
@@ -887,20 +787,29 @@ def fetch_data(
 
 
 def filter_namespaces_by_cluster_and_namespace(
-    namespaces, cluster_name, namespace_name
-):
-    if cluster_name:
-        namespaces = [n for n in namespaces if n["cluster"]["name"] == cluster_name]
+    namespaces: Sequence[dict[str, Any]],
+    cluster_names: Iterable[str] | None,
+    exclude_clusters: Iterable[str] | None,
+    namespace_name: str | None,
+) -> Sequence[dict[str, Any]]:
+    if cluster_names:
+        namespaces = [n for n in namespaces if n["cluster"]["name"] in cluster_names]
+    elif exclude_clusters:
+        namespaces = [
+            n for n in namespaces if n["cluster"]["name"] not in exclude_clusters
+        ]
+
     if namespace_name:
         namespaces = [n for n in namespaces if n["name"] == namespace_name]
+
     return namespaces
 
 
 def canonicalize_namespaces(
     namespaces: Iterable[dict[str, Any]],
-    providers: list[str],
-    resource_schema_filter: Optional[str] = None,
-) -> tuple[list[dict[str, Any]], Optional[list[str]]]:
+    providers: Sequence[str],
+    resource_schema_filter: str | None = None,
+) -> tuple[list[dict[str, Any]], list[str] | None]:
     canonicalized_namespaces = []
     override = None
     logging.debug(f"Received providers {providers}")
@@ -924,6 +833,8 @@ def canonicalize_namespaces(
                 override = ["Secret"]
             elif providers[0] == "route":
                 override = ["Route"]
+            elif providers[0] == "prometheus-rule":
+                override = ["PrometheusRule"]
 
             namespace_info["openshiftResources"] = ors
             canonicalized_namespaces.append(namespace_info)
@@ -932,16 +843,17 @@ def canonicalize_namespaces(
 
 
 def get_namespaces(
-    providers: Optional[list[str]] = None,
-    cluster_name: Optional[str] = None,
-    namespace_name: Optional[str] = None,
-    resource_schema_filter: Optional[str] = None,
-    filter_by_shard: Optional[bool] = True,
-) -> tuple[list[dict[str, Any]], Optional[list[str]]]:
+    providers: Sequence[str] | None = None,
+    cluster_names: Iterable[str] | None = None,
+    exclude_clusters: Iterable[str] | None = None,
+    namespace_name: str | None = None,
+    resource_schema_filter: str | None = None,
+    filter_by_shard: bool | None = True,
+) -> tuple[list[dict[str, Any]], list[str] | None]:
     if providers is None:
         providers = []
     gqlapi = gql.get_api()
-    namespaces = [
+    namespaces: list[dict[str, Any]] = [
         namespace_info
         for namespace_info in gqlapi.query(NAMESPACES_QUERY)["namespaces"]
         if not ob.is_namespace_deleted(namespace_info)
@@ -952,31 +864,51 @@ def get_namespaces(
             )
         )
     ]
-    namespaces = filter_namespaces_by_cluster_and_namespace(
-        namespaces, cluster_name, namespace_name
+    _namespaces = filter_namespaces_by_cluster_and_namespace(
+        namespaces, cluster_names, exclude_clusters, namespace_name
     )
-    return canonicalize_namespaces(namespaces, providers, resource_schema_filter)
+    return canonicalize_namespaces(_namespaces, providers, resource_schema_filter)
 
 
 @defer
 def run(
-    dry_run,
-    thread_pool_size=10,
-    internal=None,
-    use_jump_host=True,
-    providers=None,
-    cluster_name=None,
-    namespace_name=None,
-    init_api_resources=False,
-    defer=None,
-):
+    dry_run: bool,
+    thread_pool_size: int = 10,
+    internal: bool | None = None,
+    use_jump_host: bool = True,
+    providers: Sequence[str] | None = None,
+    cluster_name: Sequence[str] | None = None,
+    exclude_cluster: Sequence[str] | None = None,
+    namespace_name: str | None = None,
+    init_api_resources: bool = False,
+    defer: Callable | None = None,
+) -> ResourceInventory | None:
+    # https://click.palletsprojects.com/en/8.1.x/options/#multiple-options
+    cluster_names = cluster_name
+    exclude_clusters = exclude_cluster
+
+    if exclude_clusters and not dry_run:
+        raise RuntimeError("--exclude-cluster is only supported in dry-run mode")
+
+    if exclude_clusters and cluster_names:
+        raise RuntimeError(
+            "--cluster-name and --exclude-cluster can not be used together"
+        )
+    if cluster_names and len(cluster_names) > 1 and not dry_run:
+        raise RuntimeError(
+            "Running with multiple clusters is only supported in dry-run mode"
+        )
+
     namespaces, overrides = get_namespaces(
-        providers=providers, cluster_name=cluster_name, namespace_name=namespace_name
+        providers=providers,
+        cluster_names=cluster_names,
+        exclude_clusters=exclude_clusters,
+        namespace_name=namespace_name,
     )
     if not namespaces:
-        logging.info(
+        logging.debug(
             "No namespaces found when filtering for "
-            f"cluster={cluster_name}, namespace={namespace_name}. "
+            f"cluster={cluster_names}, namespace={namespace_name}. "
             "Exiting."
         )
         return None
@@ -988,12 +920,14 @@ def run(
         init_api_resources=init_api_resources,
         overrides=overrides,
     )
-    defer(oc_map.cleanup)
+    if defer:
+        defer(oc_map.cleanup)
     if dry_run and QONTRACT_INTEGRATION == "openshift-resources":
         error = check_cluster_scoped_resources(oc_map, ri, namespaces, None)
         if error:
             sys.exit(1)
 
+    ob.publish_metrics(ri, QONTRACT_INTEGRATION)
     ob.realize_data(dry_run, oc_map, ri, thread_pool_size)
 
     if ri.has_error_registered():
@@ -1015,7 +949,7 @@ class CheckNamespaceResources(Protocol):
 class CheckClusterScopedResourceNames:
     oc_map: OC_Map
     ri: ResourceInventory
-    namespaces: list[Mapping[str, Any]]
+    namespaces: Iterable[Mapping[str, Any]]
 
     def check(self) -> list[Exception]:
         errors: list[Exception] = []
@@ -1064,7 +998,7 @@ class CheckClusterScopedResourceNames:
 @dataclass
 class CheckClusterScopedResourceDuplicates:
     oc_map: OC_Map
-    all_namespaces: Optional[list[Mapping]] = None
+    all_namespaces: Iterable[Mapping] | None = None
 
     def check(self) -> list[Exception]:
         errors: list[Exception] = []
@@ -1086,13 +1020,13 @@ class CheckClusterScopedResourceDuplicates:
 
     def _find_resource_duplicates(
         self, cluster_cs_resources: dict[str, dict[str, dict[str, list[str]]]]
-    ) -> list[Tuple[str, str, str, list[str]]]:
+    ) -> list[tuple[str, str, str, list[str]]]:
         # ) -> dict[Tuple[str, str, str], list[str]]:
         """Finds cluster resource duplicates by kind/name.
         :param cluster_cs_resources
         :return: duplicates as [(cluster, kind, name, [namespaces])]
         """
-        duplicates: list[Tuple[str, str, str, list[str]]] = []
+        duplicates: list[tuple[str, str, str, list[str]]] = []
 
         for cluster, cluster_resources in cluster_cs_resources.items():
             _kind_name: dict[str, dict[str, list[str]]] = {}
@@ -1111,10 +1045,9 @@ class CheckClusterScopedResourceDuplicates:
 def check_cluster_scoped_resources(
     oc_map: OC_Map,
     ri: ResourceInventory,
-    namespaces: list[Mapping[str, Any]],
-    all_namespaces: Optional[list[Mapping[str, Any]]] = None,
+    namespaces: Iterable[Mapping[str, Any]],
+    all_namespaces: Iterable[Mapping[str, Any]] | None = None,
 ) -> bool:
-
     checks = [
         CheckClusterScopedResourceNames(oc_map, ri, namespaces),
         CheckClusterScopedResourceDuplicates(oc_map, all_namespaces),
@@ -1134,7 +1067,7 @@ def check_cluster_scoped_resources(
 def get_cluster_scoped_resources(
     oc_map: OC_Map,
     clusters: Iterable[str],
-    namespaces: Optional[Iterable[Mapping[str, Any]]] = None,
+    namespaces: Iterable[Mapping[str, Any]] | None = None,
     thread_pool_size: int = 10,
 ) -> dict[str, dict[str, dict[str, list[str]]]]:
     """Returns cluster scoped resources for a list of clusters
@@ -1171,7 +1104,7 @@ def get_cluster_scoped_resources(
 def _get_namespace_cluster_scoped_resources(
     namespace: Mapping,
     oc_map: OC_Map,
-) -> Tuple[str, str, dict[str, dict[str, Any]]]:
+) -> tuple[str, str, dict[str, dict[str, Any]]]:
     """Returns all non-namespaced resources defined in a namespace manifest.
 
     :param namespace: the namespace dict
@@ -1190,7 +1123,7 @@ def _get_namespace_cluster_scoped_resources(
 
 
 def early_exit_desired_state(
-    providers: list[str], resource_schema_filter: Optional[str] = None
+    providers: list[str], resource_schema_filter: str | None = None
 ) -> dict[str, Any]:
     settings = queries.get_secret_reader_settings()
     namespaces, _ = get_namespaces(
@@ -1212,20 +1145,28 @@ def early_exit_desired_state(
             settings=settings,
         )
 
-    def post_process_ns(ns):
-        ns[IDENTIFIER_FIELD_NAME] = f"{ns['cluster']['name']}/{ns['name']}"
+    def post_process_ns(ns: MutableMapping) -> MutableMapping:
         # the sharedResources have been aggreated into the openshiftResources
         # and are no longer needed - speeds up diffing process
         del ns["sharedResources"]
         return ns
 
+    # assemble all namespaces and resources file under a cluster
+    state_for_clusters = defaultdict(list)
+    for ns in namespaces:
+        state_for_clusters[ns["cluster"]["name"]].append(post_process_ns(ns))
+    for res in resources:
+        state_for_clusters[res["cluster"]].append(res)
+
     return {
-        "namespaces": [post_process_ns(ns) for ns in namespaces],
-        "resources": resources,
+        "state": {
+            cluster: {"shard": cluster, "hash": DeepHash(state).get(state)}
+            for cluster, state in state_for_clusters.items()
+        }
     }
 
 
-def _early_exit_fetch_resource(spec, settings):
+def _early_exit_fetch_resource(spec: Sequence, settings: Mapping) -> dict[str, str]:
     resource = spec[0]
     ns_info = spec[1]
     cluster_name = ns_info["cluster"]["name"]
@@ -1253,47 +1194,82 @@ def _early_exit_fetch_resource(spec, settings):
 
 
 @contextmanager
-def early_exit_monkey_patch():
+def early_exit_monkey_patch() -> Generator:
     """Avoid looking outside of app-interface on early-exit pr-check."""
-    orig_lookup_secret = lookup_secret
-    orig_lookup_github_file_content = lookup_github_file_content
-    orig_url_makes_sense = url_makes_sense
-    orig_check_alertmanager_config = check_alertmanager_config
-
-    try:
-        yield _early_exit_monkey_patch_assign(
-            lambda path, key, version=None, tvars=None, settings=None: f"vault({path}, {key}, {version})",
-            lambda repo, path, ref, tvars=None, settings=None: f"github({repo}, {path}, {ref})",
-            lambda url: False,
-            lambda data, path, alertmanager_config_key, decode_base64=False: True,
+    with patch.multiple(
+        jinja2_utils,
+        lookup_secret=DEFAULT,
+        lookup_github_file_content=DEFAULT,
+        url_makes_sense=DEFAULT,
+        lookup_s3_object=DEFAULT,
+        list_s3_objects=DEFAULT,
+    ) as mocks:
+        # mock lookup_secret
+        mocks["lookup_secret"].side_effect = (
+            lambda path,
+            key,
+            version=None,
+            tvars=None,
+            allow_not_found=False,
+            settings=None,
+            secret_reader=None: f"vault({path}, {key}, {version}"
         )
-    finally:
-        _early_exit_monkey_patch_assign(
-            orig_lookup_secret,
-            orig_lookup_github_file_content,
-            orig_url_makes_sense,
-            orig_check_alertmanager_config,
+        # needed for jinja2 `is_safe_callable`
+        mocks["lookup_secret"].unsafe_callable = False
+        mocks["lookup_secret"].alters_data = False
+
+        # mock lookup_github_file_content
+        mocks["lookup_github_file_content"].side_effect = (
+            lambda repo,
+            path,
+            ref,
+            tvars=None,
+            settings=None,
+            secret_reader=None: f"github({repo}, {path}, {ref})"
         )
+        # needed for jinja2 `is_safe_callable`
+        mocks["lookup_github_file_content"].unsafe_callable = False
+        mocks["lookup_github_file_content"].alters_data = False
+
+        # mock url_makes_sense
+        mocks["url_makes_sense"].side_effect = lambda url: False
+        # needed for jinja2 `is_safe_callable`
+        mocks["url_makes_sense"].unsafe_callable = False
+        mocks["url_makes_sense"].alters_data = False
+
+        # mock lookup_s3_object
+        mocks["lookup_s3_object"].side_effect = (
+            lambda account_name,
+            bucket_name,
+            path,
+            region_name=None: f"lookup_s3_object({account_name}, {bucket_name}, {path}, {region_name})"
+        )
+        # needed for jinja2 `is_safe_callable`
+        mocks["lookup_s3_object"].unsafe_callable = False
+        mocks["lookup_s3_object"].alters_data = False
+
+        # mock list_s3_objects
+        mocks["list_s3_objects"].side_effect = (
+            lambda account_name,
+            bucket_name,
+            path,
+            region_name=None: f"list_s3_objects({account_name}, {bucket_name}, {path}, {region_name})"
+        )
+        # needed for jinja2 `is_safe_callable`
+        mocks["list_s3_objects"].unsafe_callable = False
+        mocks["list_s3_objects"].alters_data = False
 
-
-def _early_exit_monkey_patch_assign(
-    lookup_secret,
-    lookup_github_file_content,
-    url_makes_sense,
-    check_alertmanager_config,
-):
-    sys.modules[__name__].lookup_secret = lookup_secret
-    sys.modules[__name__].lookup_github_file_content = lookup_github_file_content
-    sys.modules[__name__].url_makes_sense = url_makes_sense
-    sys.modules[__name__].check_alertmanager_config = check_alertmanager_config
+        with patch(
+            "reconcile.openshift_resources_base.check_alertmanager_config",
+            return_value=True,
+        ):
+            yield
 
 
 def desired_state_shard_config() -> DesiredStateShardConfig:
     return DesiredStateShardConfig(
         shard_arg_name="cluster_name",
-        shard_path_selectors={
-            "namespaces[*].cluster.name",
-            "resources[*].cluster",
-        },
+        shard_arg_is_collection=True,
+        shard_path_selectors={"state.*.shard"},
         sharded_run_review=lambda proposal: len(proposal.proposed_shards) <= 2,
     )