qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/closedbox_endpoint_monitoring_base.py

@@ -3,10 +3,7 @@ import logging
 from collections import defaultdict
 from collections.abc import Callable
 from dataclasses import field
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 from urllib.parse import urlparse
 
 from pydantic.dataclasses import dataclass
@@ -28,7 +25,6 @@ class ClosedBoxReconcilerError(Exception):
 
 @dataclass(frozen=True, eq=True)
 class BlackboxMonitoringProvider:
-
     module: str
     # the namespace of a blackbox-exporter provider is mapped as dict
     # since its only use with ob.fetch_current_state is as a dict
@@ -38,7 +34,6 @@ class BlackboxMonitoringProvider:
 
 @dataclass(frozen=True, eq=True)
 class SignalfxMonitoringProvier:
-
     # the namespace of a signalfx provider is mapped as dict
     # since its only use with ob.fetch_current_state is as a dict
     namespace: dict[str, Any] = field(compare=False, hash=False)
@@ -48,18 +43,17 @@ class SignalfxMonitoringProvier:
 
 @dataclass(frozen=True, eq=True)
 class EndpointMonitoringProvider:
-
     name: str
     provider: str
     description: str
-    timeout: Optional[str] = None
-    checkInterval: Optional[str] = None
-    blackboxExporter: Optional[BlackboxMonitoringProvider] = None
-    signalFx: Optional[SignalfxMonitoringProvier] = None
-    metricLabels: Optional[str] = None
+    timeout: str | None = None
+    checkInterval: str | None = None
+    blackboxExporter: BlackboxMonitoringProvider | None = None
+    signalFx: SignalfxMonitoringProvier | None = None
+    metricLabels: str | None = None
 
     @property
-    def namespace(self) -> Optional[dict[str, Any]]:
+    def namespace(self) -> dict[str, Any] | None:
         if self.blackboxExporter:
             return self.blackboxExporter.namespace
 
@@ -75,14 +69,12 @@ class EndpointMonitoringProvider:
 
 @dataclass
 class Endpoint:
-
     name: str
     description: str
     url: str
 
     @dataclass
     class Monitoring:
-
         provider: EndpointMonitoringProvider
 
     monitoring: list[Monitoring]
@@ -90,7 +82,7 @@ class Endpoint:
 
 def parse_prober_url(url: str) -> dict[str, str]:
     parsed_url = urlparse(url)
-    if parsed_url.scheme not in ["http", "https"]:
+    if parsed_url.scheme not in {"http", "https"}:
         raise ValueError(
             "the prober URL needs to be an http:// or https:// one " f"but is {url}"
         )
@@ -136,7 +128,7 @@ def fill_desired_state(
 def run_for_provider(
     provider: str,
     probe_builder: Callable[
-        [EndpointMonitoringProvider, list[Endpoint]], Optional[OpenshiftResource]
+        [EndpointMonitoringProvider, list[Endpoint]], OpenshiftResource | None
     ],
     integration: str,
     integration_version: str,
@@ -167,6 +159,7 @@ def run_for_provider(
             probe = probe_builder(ep_mon_provider, endpoints)
             if probe:
                 fill_desired_state(ep_mon_provider, probe, ri)
+        ob.publish_metrics(ri, integration)
         ob.realize_data(dry_run, oc_map, ri, thread_pool_size, recycle_pods=False)
 
         if ri.has_error_registered():

reconcile/cluster_auth_rhidp/integration.py

@@ -0,0 +1,257 @@
+import logging
+from collections.abc import Callable, Iterable
+from typing import Any
+
+from deepdiff import DeepHash
+
+from reconcile.gql_definitions.cluster_auth_rhidp.clusters import (
+    ClusterAuthRHIDPV1,
+    ClusterV1,
+)
+from reconcile.gql_definitions.cluster_auth_rhidp.clusters import query as cluster_query
+from reconcile.gql_definitions.common.ocm_environments import (
+    query as ocm_environment_query,
+)
+from reconcile.gql_definitions.fragments.ocm_environment import OCMEnvironment
+from reconcile.rhidp.common import (
+    AUTH_NAME_LABEL_KEY,
+    ISSUER_LABEL_KEY,
+    RHIDP_NAMESPACE_LABEL_KEY,
+    STATUS_LABEL_KEY,
+    StatusValue,
+)
+from reconcile.utils import gql
+from reconcile.utils.defer import defer
+from reconcile.utils.differ import diff_mappings
+from reconcile.utils.disabled_integrations import integration_is_enabled
+from reconcile.utils.ocm.clusters import discover_clusters_for_organizations
+from reconcile.utils.ocm.label_sources import ClusterRef, LabelState
+from reconcile.utils.ocm.labels import add_label, delete_label, update_label
+from reconcile.utils.ocm_base_client import (
+    OCMAPIClientConfigurationProtocol,
+    OCMBaseClient,
+    init_ocm_base_client,
+)
+from reconcile.utils.runtime.integration import (
+    PydanticRunParams,
+    QontractReconcileIntegration,
+)
+from reconcile.utils.secret_reader import SecretReaderBase
+from reconcile.utils.semver_helper import make_semver
+
+QONTRACT_INTEGRATION = "cluster-auth-rhidp"
+QONTRACT_INTEGRATION_VERSION = make_semver(1, 0, 0)
+
+
+class ClusterAuthRhidpIntegrationParams(PydanticRunParams):
+    pass
+
+
+class ManagedLabelConflictError(Exception):
+    pass
+
+
+OcmApis = dict[str, OCMBaseClient]
+
+
+class ClusterAuthRhidpIntegration(
+    QontractReconcileIntegration[ClusterAuthRhidpIntegrationParams]
+):
+    """Manages the OCM subscription labels for clusters with RHIDP authentication."""
+
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION
+
+    def get_early_exit_desired_state(
+        self, query_func: Callable | None = None
+    ) -> dict[str, Any]:
+        """Return the desired state for early exit."""
+        if not query_func:
+            query_func = gql.get_api().query
+
+        desired = {
+            "subs_labels": self.fetch_desired_state(self.get_clusters(query_func)),
+        }
+        # to figure out wheter to run a PR check of to exit early, a hash value
+        # of the desired state is sufficient
+        return {"hash": DeepHash(desired).get(desired)}
+
+    def get_clusters(self, query_func: Callable) -> list[ClusterV1]:
+        data = cluster_query(query_func)
+        return [
+            c
+            for c in data.clusters or []
+            if integration_is_enabled(self.name, c)
+            # ocm is mandatory
+            and c.ocm
+            and c.spec
+            and c.spec.q_id
+        ]
+
+    def get_environments(self, query_func: Callable) -> list[OCMEnvironment]:
+        return ocm_environment_query(query_func).environments
+
+    def fetch_desired_state(self, clusters: Iterable[ClusterV1]) -> LabelState:
+        """Fetches the desired state of subscription labels for the given clusters."""
+        states: LabelState = {}
+
+        for cluster in clusters:
+            assert cluster.ocm and cluster.spec and cluster.spec.q_id  # make mypy happy
+            label = {}
+            if auth := next(
+                (
+                    a
+                    for a in cluster.auth
+                    if isinstance(a, ClusterAuthRHIDPV1) and a.service == "rhidp"
+                ),
+                None,
+            ):
+                label = {
+                    STATUS_LABEL_KEY: auth.status or StatusValue.ENABLED.value,
+                    AUTH_NAME_LABEL_KEY: auth.name,
+                }
+                if auth.issuer:
+                    label[ISSUER_LABEL_KEY] = auth.issuer
+            cluster_ref = ClusterRef(
+                cluster_id=cluster.spec.q_id,
+                org_id=cluster.ocm.org_id,
+                ocm_env=cluster.ocm.environment.name,
+                name=cluster.name,
+                label_container_href=None,
+            )
+            states[cluster_ref] = label
+
+        return states
+
+    def fetch_current_state(
+        self,
+        ocm_apis: OcmApis,
+        clusters: Iterable[ClusterV1],
+        managed_label_prefixes: Iterable[str],
+    ) -> LabelState:
+        """Fetches the current state of subscription labels for the given clusters.
+
+        If a cluster can't be found in OCM, the resulting dict will not contain a
+        state for it, not even an empty one.
+        """
+        cluster_ids = {c.spec.q_id for c in clusters if c.spec and c.spec.q_id}
+        states: LabelState = {}
+        for env_name, ocm_api in ocm_apis.items():
+            for cluster_details in discover_clusters_for_organizations(
+                ocm_api=ocm_api,
+                organization_ids=list({
+                    c.ocm.org_id
+                    for c in clusters
+                    if c.ocm and c.ocm.environment.name == env_name
+                }),
+            ):
+                if cluster_details.ocm_cluster.id not in cluster_ids:
+                    # there might be more clusters in an organization than we care about
+                    continue
+
+                filtered_labels = {
+                    label: value
+                    for label, value in cluster_details.subscription_labels.get_values_dict().items()
+                    if label.startswith(tuple(managed_label_prefixes))
+                }
+                states[
+                    ClusterRef(
+                        cluster_id=cluster_details.ocm_cluster.id,
+                        org_id=cluster_details.organization_id,
+                        ocm_env=env_name,
+                        name=cluster_details.ocm_cluster.name,
+                        label_container_href=f"{cluster_details.ocm_cluster.subscription.href}/labels",
+                    )
+                ] = filtered_labels
+        return states
+
+    def init_ocm_apis(
+        self,
+        environments: Iterable[OCMEnvironment],
+        init_ocm_base_client: Callable[
+            [OCMAPIClientConfigurationProtocol, SecretReaderBase], OCMBaseClient
+        ] = init_ocm_base_client,
+    ) -> OcmApis:
+        """Initialize OCM clients for each OCM environment."""
+        return {
+            env.name: init_ocm_base_client(env, self.secret_reader)
+            for env in environments
+        }
+
+    def reconcile(
+        self,
+        dry_run: bool,
+        ocm_apis: OcmApis,
+        current_state: LabelState,
+        desired_state: LabelState,
+    ) -> None:
+        # we iterate via the current state because it refers to the clusters we can act on
+        for label_owner_ref, current_labels in current_state.items():
+            ocm_api = ocm_apis[label_owner_ref.ocm_env]
+            desired_labels = desired_state.get(label_owner_ref, {})
+            if current_labels == desired_labels:
+                continue
+
+            diff_result = diff_mappings(current_labels, desired_labels)
+
+            for label_to_add, value in diff_result.add.items():
+                logging.info([
+                    "create_label",
+                    *label_owner_ref.identity_labels(),
+                    f"{label_to_add}={value}",
+                ])
+                if not dry_run:
+                    add_label(
+                        ocm_api=ocm_api,
+                        label_container_href=label_owner_ref.required_label_container_href(),
+                        label=label_to_add,
+                        value=value,
+                    )
+            for label_to_rm, value in diff_result.delete.items():
+                logging.info([
+                    "delete_label",
+                    *label_owner_ref.identity_labels(),
+                    f"{label_to_rm}={value}",
+                ])
+                if not dry_run:
+                    delete_label(
+                        ocm_api=ocm_api,
+                        label_container_href=label_owner_ref.required_label_container_href(),
+                        label=label_to_rm,
+                    )
+            for label_to_update, diff_pair in diff_result.change.items():
+                value = diff_pair.desired
+                logging.info([
+                    "update_label",
+                    *label_owner_ref.identity_labels(),
+                    f"{label_to_update}={value}",
+                ])
+                if not dry_run:
+                    update_label(
+                        ocm_api=ocm_api,
+                        label_container_href=label_owner_ref.required_label_container_href(),
+                        label=label_to_update,
+                        value=value,
+                    )
+
+    @defer
+    def run(self, dry_run: bool, defer: Callable | None = None) -> None:
+        """Run the integration."""
+        gql_api = gql.get_api()
+        clusters = self.get_clusters(gql_api.query)
+        environments = self.get_environments(gql_api.query)
+        ocm_apis = self.init_ocm_apis(environments, init_ocm_base_client)
+        if defer:
+            defer(lambda: [ocm_api.close() for ocm_api in ocm_apis.values()])  # type: ignore
+
+        current_state = self.fetch_current_state(
+            ocm_apis, clusters, managed_label_prefixes=[RHIDP_NAMESPACE_LABEL_KEY]
+        )
+        desired_state = self.fetch_desired_state(clusters)
+        self.reconcile(
+            dry_run=dry_run,
+            ocm_apis=ocm_apis,
+            current_state=current_state,
+            desired_state=desired_state,
+        )

reconcile/cluster_deployment_mapper.py

@@ -1,9 +1,6 @@
 import logging
 import sys
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 from reconcile.status import ExitCodes
 from reconcile.typed_queries.app_interface_vault_settings import (
@@ -18,7 +15,7 @@ from reconcile.utils.vault import VaultClient
 QONTRACT_INTEGRATION = "cluster-deployment-mapper"
 
 
-def run(dry_run: bool, vault_output_path: Optional[str]) -> None:
+def run(dry_run: bool, vault_output_path: str | None) -> None:
     """Get Hive ClusterDeployments from clusters and save mapping to Vault"""
     if not vault_output_path:
         logging.error("must supply vault output path")

reconcile/cna/assets/asset.py

@@ -9,10 +9,7 @@ from dataclasses import (
     field,
 )
 from enum import Enum
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 
 class AssetError(Exception):
@@ -31,9 +28,9 @@ class AssetStatus(Enum):
 
 @dataclass(frozen=True)
 class Asset(ABC):
-    uuid: Optional[str] = field(compare=False, hash=True)
-    href: Optional[str] = field(compare=False, hash=True)
-    status: Optional[AssetStatus] = field(compare=False, hash=True)
+    uuid: str | None = field(compare=False, hash=True)
+    href: str | None = field(compare=False, hash=True)
+    status: AssetStatus | None = field(compare=False, hash=True)
     name: str
     kind: AssetType
 

reconcile/cna/assets/null.py

@@ -2,10 +2,7 @@ from __future__ import annotations
 
 from collections.abc import Mapping
 from dataclasses import dataclass
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 from reconcile.cna.assets.asset import (
     Asset,
@@ -18,7 +15,7 @@ from reconcile.gql_definitions.cna.queries.cna_resources import CNANullAssetV1
 
 @dataclass(frozen=True)
 class NullAsset(Asset):
-    addr_block: Optional[str]
+    addr_block: str | None
 
     def api_payload(self) -> dict[str, Any]:
         return {

reconcile/cna/integration.py

@@ -3,7 +3,6 @@ from collections.abc import (
     Iterable,
     Mapping,
 )
-from typing import Optional
 
 from reconcile.cna.assets.asset_factory import asset_factory_from_schema
 from reconcile.cna.client import CNAClient
@@ -45,8 +44,8 @@ class CNAIntegration:
         self,
         cna_clients: Mapping[str, CNAClient],
         namespaces: Iterable[NamespaceV1],
-        desired_states: Optional[Mapping[str, State]] = None,
-        current_states: Optional[Mapping[str, State]] = None,
+        desired_states: Mapping[str, State] | None = None,
+        current_states: Mapping[str, State] | None = None,
     ):
         self._cna_clients = cna_clients
         self._namespaces = namespaces

reconcile/cna/state.py

@@ -4,10 +4,7 @@ from collections.abc import (
     Iterable,
     Mapping,
 )
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 from reconcile.cna.assets.asset import (
     Asset,
@@ -29,7 +26,7 @@ class State:
     deletions and updates to reach another state.
     """
 
-    def __init__(self, assets: Optional[dict[AssetType, dict[str, Asset]]] = None):
+    def __init__(self, assets: dict[AssetType, dict[str, Asset]] | None = None):
         self._assets: dict[AssetType, dict[str, Asset]] = {}
         for kind in AssetType:
             self._assets[kind] = {}
@@ -39,10 +36,10 @@ class State:
     def __eq__(self, other: object) -> bool:
         if not isinstance(other, State):
             return False
-        if not set(list(self._assets.keys())) == set(list(other._assets.keys())):
+        if not set(self._assets.keys()) == set(other._assets.keys()):
             return False
         for kind in list(self._assets.keys()):
-            if not set(list(self._assets[kind])) == set(list(other._assets[kind])):
+            if not set(self._assets[kind]) == set(other._assets[kind]):
                 return False
             for name, asset in self._assets[kind].items():
                 if asset != other._assets[kind][name]:
@@ -85,7 +82,7 @@ class State:
                 if asset_name not in self._assets[kind]:
                     continue
                 asset = self._assets[kind][asset_name]
-                if asset.status in (AssetStatus.TERMINATED, AssetStatus.PENDING):
+                if asset.status in {AssetStatus.TERMINATED, AssetStatus.PENDING}:
                     continue
                 if asset == other_asset:
                     # There is no diff - no need to update
@@ -106,7 +103,7 @@ class State:
         ans = State()
         for kind in AssetType:
             for asset_name, asset in self._assets[kind].items():
-                if asset.status in (AssetStatus.TERMINATED, AssetStatus.PENDING):
+                if asset.status in {AssetStatus.TERMINATED, AssetStatus.PENDING}:
                     continue
                 if other_asset := other._assets[kind].get(asset_name):
                     if other_asset.status == AssetStatus.TERMINATED:

reconcile/dashdotdb_base.py

@@ -1,11 +1,9 @@
 import logging
 import os
+from base64 import b64encode
 from collections.abc import Mapping
 from dataclasses import dataclass
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 from urllib.parse import urljoin
 
 import requests
@@ -22,8 +20,8 @@ LOG = logging.getLogger(__name__)
 class DashdotDBSecret:
     path: str
     field: str
-    q_format: Optional[str]
-    version: Optional[int]
+    q_format: str | None
+    version: int | None
 
 
 DASHDOTDB_SECRET = DashdotDBSecret(
@@ -52,6 +50,7 @@ class DashdotdbBase:
         self.dashdotdb_pass = self.secret_content["password"]
         self.logmarker = marker
         self.scope = scope
+        self.dashdotdb_token: str | None = None
 
     def _get_token(self) -> None:
         if self.dry_run:
@@ -105,10 +104,26 @@ class DashdotdbBase:
         data: Mapping[Any, Any],
         timeout: tuple[int, int] = (5, 120),
     ) -> requests.Response:
+        headers: dict[str, str] = {}
+        if self.dashdotdb_token:
+            headers["X-Auth"] = str(self.dashdotdb_token)
         return requests.post(
             url=endpoint,
             json=data,
-            headers={"X-Auth": self.dashdotdb_token},
+            headers=headers,
+            auth=(self.dashdotdb_user, self.dashdotdb_pass),
+            timeout=timeout,
+        )
+
+    def _do_get(
+        self,
+        endpoint: str,
+        params: Mapping[Any, Any],
+        timeout: tuple[int, int] = (5, 120),
+    ) -> requests.Response:
+        return requests.get(
+            url=endpoint,
+            params=params,
             auth=(self.dashdotdb_user, self.dashdotdb_pass),
             timeout=timeout,
         )
@@ -116,8 +131,10 @@ class DashdotdbBase:
     def _promget(
         self,
         url: str,
-        params: Optional[Mapping[Any, Any]],
-        token: Optional[str] = None,
+        params: Mapping[Any, Any] | None,
+        token: str | None = None,
+        username: str | None = None,
+        password: str | None = None,
         ssl_verify: bool = True,
         uri: str = "api/v1/query",
     ) -> dict[Any, Any]:
@@ -128,8 +145,12 @@ class DashdotdbBase:
         }
         if token:
             headers["Authorization"] = f"Bearer {token}"
+        elif username and password:
+            headers["Authorization"] = (
+                f"Basic {b64encode(f'{username}:{password}'.encode()).decode('utf-8')}"
+            )
         response = requests.get(
-            url, params=params, headers=headers, verify=ssl_verify, timeout=(5, 120)
+            url, params=params, headers=headers, verify=ssl_verify, timeout=(5, 300)
         )
         response.raise_for_status()
 

reconcile/dashdotdb_cso.py

@@ -1,7 +1,4 @@
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 import requests
 from sretoolbox.utils import threaded
@@ -44,7 +41,7 @@ class DashdotdbCSO(DashdotdbBase):
             secret_reader=secret_reader,
         )
 
-    def _post(self, manifest: dict[Any, Any]) -> Optional[requests.Response]:
+    def _post(self, manifest: dict[Any, Any]) -> requests.Response | None:
         if manifest is None:
             return None
 
@@ -70,7 +67,7 @@ class DashdotdbCSO(DashdotdbBase):
         return response
 
     @staticmethod
-    def _get_imagemanifestvuln(cluster: str, oc_map: OCMap) -> Optional[dict[str, Any]]:
+    def _get_imagemanifestvuln(cluster: str, oc_map: OCMap) -> dict[str, Any] | None:
         LOG.info("%s processing %s", LOGMARKER, cluster)
         oc = oc_map.get(cluster)
         if isinstance(oc, OCLogMsg):