qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/rhidp/sso_client/base.py

@@ -0,0 +1,266 @@
+import logging
+from collections.abc import (
+    Iterable,
+    Sequence,
+)
+from urllib.parse import (
+    urljoin,
+    urlparse,
+    urlunparse,
+)
+
+import jwt
+
+from reconcile.rhidp.common import (
+    Cluster,
+    cluster_vault_secret,
+    cluster_vault_secret_id,
+    expose_base_metrics,
+)
+from reconcile.rhidp.sso_client.metrics import (
+    RhIdpSSOClientCounter,
+    RhIdpSSOClientIatExpiration,
+    RhIdpSSOClientReconcileCounter,
+    RhIdpSSOClientReconcileErrorCounter,
+)
+from reconcile.utils import metrics
+from reconcile.utils.keycloak import (
+    KeycloakInstance,
+    KeycloakMap,
+    SSOClient,
+)
+from reconcile.utils.secret_reader import VaultSecretReader
+
+DesiredSSOClients = dict[str, Cluster]
+
+
+def console_url_to_oauth_url(console_url: str, auth_name: str) -> str:
+    """Convert a console URL to an OAuth callback URL."""
+    if console_url.startswith("https://console-openshift-console.apps.rosa."):
+        # ROSA cluster
+
+        url = urlparse(
+            urljoin(
+                console_url.replace("console-openshift-console.apps.rosa", "oauth"),
+                f"/oauth2callback/{auth_name}",
+            )
+        )
+        if url.port is None:
+            url = url._replace(netloc=url.netloc + ":443")
+        return urlunparse(url)
+    # OSD cluster
+    return urljoin(
+        console_url.replace("console-openshift-console", "oauth-openshift"),
+        f"/oauth2callback/{auth_name}",
+    )
+
+
+def run(
+    integration_name: str,
+    ocm_environment: str,
+    clusters: Iterable[Cluster],
+    secret_reader: VaultSecretReader,
+    keycloak_vault_paths: Iterable[str],
+    vault_input_path: str,
+    contacts: Sequence[str],
+    dry_run: bool,
+) -> None:
+    with metrics.transactional_metrics(ocm_environment) as metrics_container:
+        # metrics
+        expose_base_metrics(
+            metrics_container, integration_name, ocm_environment, clusters
+        )
+
+        # APIs
+        keycloak_instances: list[KeycloakInstance] = []
+        for path in keycloak_vault_paths:
+            secret = secret_reader.read_all({"path": path})
+            iat = secret["initial-access-token"]
+            token = jwt.decode(
+                iat,
+                "secret",
+                algorithms=["HS256"],
+                options={"verify_signature": False},
+            )
+            metrics_container.set_gauge(
+                RhIdpSSOClientIatExpiration(
+                    integration=integration_name,
+                    ocm_environment=ocm_environment,
+                    path=path,
+                ),
+                value=token["exp"],
+            )
+            keycloak_instances.append(
+                KeycloakInstance(
+                    url=secret["url"],
+                    initial_access_token=iat,
+                )
+            )
+        keycloak_map = KeycloakMap(keycloak_instances)
+
+        # run
+        try:
+            existing_sso_client_ids = fetch_current_state(
+                secret_reader=secret_reader, vault_input_path=vault_input_path
+            )
+            metrics_container.set_gauge(
+                RhIdpSSOClientCounter(
+                    integration=integration_name,
+                    ocm_environment=ocm_environment,
+                ),
+                value=len(existing_sso_client_ids),
+            )
+            desired_sso_clients = fetch_desired_state(clusters=clusters)
+            act(
+                keycloak_map=keycloak_map,
+                existing_sso_client_ids=existing_sso_client_ids,
+                desired_sso_clients=desired_sso_clients,
+                contacts=contacts,
+                secret_reader=secret_reader,
+                vault_input_path=vault_input_path,
+                dry_run=dry_run,
+            )
+            metrics_container.inc_counter(
+                RhIdpSSOClientReconcileCounter(
+                    integration=integration_name, ocm_environment=ocm_environment
+                )
+            )
+        except Exception:
+            metrics_container.inc_counter(
+                RhIdpSSOClientReconcileErrorCounter(
+                    integration=integration_name, ocm_environment=ocm_environment
+                )
+            )
+            raise
+
+
+def fetch_current_state(
+    secret_reader: VaultSecretReader, vault_input_path: str
+) -> list[str]:
+    """Fetch all existing SSO client IDs from vault."""
+    return secret_reader.vault_client.list(vault_input_path)  # type: ignore[attr-defined] # mypy doesn't recognize the VaultClient.__new__ method
+
+
+def fetch_desired_state(
+    clusters: Iterable[Cluster],
+) -> DesiredSSOClients:
+    """Compile all desired SSO clients from the given clusters."""
+    desired_sso_clients = {}
+    for cluster in clusters:
+        if not cluster.auth.rhidp_enabled:
+            continue
+
+        cid = cluster_vault_secret_id(
+            org_id=cluster.organization_id,
+            cluster_name=cluster.name,
+            auth_name=cluster.auth.name,
+            issuer_url=cluster.auth.issuer,
+        )
+        desired_sso_clients[cid] = cluster
+    return desired_sso_clients
+
+
+def act(
+    keycloak_map: KeycloakMap,
+    secret_reader: VaultSecretReader,
+    vault_input_path: str,
+    existing_sso_client_ids: list[str],
+    desired_sso_clients: DesiredSSOClients,
+    contacts: Sequence[str],
+    dry_run: bool,
+) -> None:
+    """Act on the difference between the current and desired state."""
+    sso_client_ids_to_remove = set(existing_sso_client_ids) - set(desired_sso_clients)
+    sso_client_ids_to_add = set(desired_sso_clients) - set(existing_sso_client_ids)
+
+    for sso_client_id in sso_client_ids_to_remove:
+        logging.info(["delete_sso_client", sso_client_id])
+        if not dry_run:
+            delete_sso_client(
+                keycloak_map=keycloak_map,
+                sso_client_id=sso_client_id,
+                secret_reader=secret_reader,
+                vault_input_path=vault_input_path,
+            )
+
+    for sso_client_id in sso_client_ids_to_add:
+        cluster = desired_sso_clients[sso_client_id]
+        logging.info([
+            "create_sso_client",
+            cluster.name,
+            cluster.auth.name,
+            sso_client_id,
+        ])
+        if not dry_run:
+            create_sso_client(
+                keycloak_map=keycloak_map,
+                sso_client_id=sso_client_id,
+                cluster=cluster,
+                contacts=contacts,
+                secret_reader=secret_reader,
+                vault_input_path=vault_input_path,
+            )
+
+
+def create_sso_client(
+    keycloak_map: KeycloakMap,
+    sso_client_id: str,
+    cluster: Cluster,
+    contacts: Sequence[str],
+    secret_reader: VaultSecretReader,
+    vault_input_path: str,
+) -> None:
+    """Create an SSO client and store SSO client data in Vault."""
+    if not cluster.console_url:
+        logging.error(
+            f"Cluster {cluster.name} does not have a console URL; maybe not ready yet. Skipping for now."
+        )
+        return
+    keycloak_api = keycloak_map.get(cluster.auth.issuer)
+    sso_client = keycloak_api.register_client(
+        client_name=sso_client_id,
+        redirect_uris=[
+            console_url_to_oauth_url(
+                console_url=cluster.console_url,
+                auth_name=cluster.auth.name,
+            )
+        ],
+        initiate_login_uri=cluster.console_url,
+        request_uris=[cluster.console_url],
+        contacts=contacts,
+    )
+    secret = cluster_vault_secret(
+        vault_input_path=vault_input_path,
+        vault_secret_id=sso_client_id,
+    )
+
+    secret_reader.vault_client.write(  # type: ignore[attr-defined] # mypy doesn't recognize the VaultClient.__new__ method
+        secret={
+            "path": secret.path,
+            "data": sso_client.dict(),
+        },
+        decode_base64=False,
+    )
+
+
+def delete_sso_client(
+    keycloak_map: KeycloakMap,
+    sso_client_id: str,
+    secret_reader: VaultSecretReader,
+    vault_input_path: str,
+) -> None:
+    """Delete an SSO client and the stored SSO client data."""
+    secret = cluster_vault_secret(
+        vault_input_path=vault_input_path,
+        vault_secret_id=sso_client_id,
+    )
+    sso_client = SSOClient(**secret_reader.read_all_secret(secret=secret))
+    keycloak_api = keycloak_map.get(sso_client.issuer)
+    keycloak_api.delete_client(
+        registration_client_uri=sso_client.registration_client_uri,
+        registration_access_token=sso_client.registration_access_token,
+    )
+
+    secret_reader.vault_client.delete(  # type: ignore[attr-defined] # mypy doesn't recognize the VaultClient.__new__ method
+        path=secret.path
+    )

reconcile/rhidp/sso_client/integration.py

@@ -0,0 +1,60 @@
+from reconcile.rhidp.common import (
+    build_cluster_objects,
+    discover_clusters,
+    get_ocm_environments,
+    get_ocm_orgs_from_env,
+)
+from reconcile.rhidp.sso_client.base import run
+from reconcile.utils.ocm_base_client import init_ocm_base_client
+from reconcile.utils.runtime.integration import (
+    PydanticRunParams,
+    QontractReconcileIntegration,
+)
+from reconcile.utils.secret_reader import VaultSecretReader
+
+QONTRACT_INTEGRATION = "rhidp-sso-client"
+
+
+class SSOClientParams(PydanticRunParams):
+    keycloak_vault_paths: list[str]
+    vault_input_path: str
+    ocm_environment: str | None = None
+    default_auth_name: str
+    default_auth_issuer_url: str
+    contacts: list[str]
+
+
+class SSOClient(QontractReconcileIntegration[SSOClientParams]):
+    """The RHIDP SSO Client integration manages SSO clients and uses OCM labels to discover clusters."""
+
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION
+
+    def run(self, dry_run: bool) -> None:
+        secret_reader = VaultSecretReader()
+        for ocm_env in get_ocm_environments(self.params.ocm_environment):
+            ocm_api = init_ocm_base_client(ocm_env, self.secret_reader)
+            cluster_details = discover_clusters(
+                ocm_api=ocm_api,
+                org_ids={
+                    org.org_id for org in get_ocm_orgs_from_env(ocm_env.name, self.name)
+                },
+            )
+            clusters = build_cluster_objects(
+                cluster_details=cluster_details,
+                default_auth_name=self.params.default_auth_name,
+                default_issuer_url=self.params.default_auth_issuer_url,
+            )
+            run(
+                integration_name=self.name,
+                ocm_environment=ocm_env.name,
+                clusters=clusters,
+                secret_reader=secret_reader,
+                keycloak_vault_paths=self.params.keycloak_vault_paths,
+                # put secrets in a subpath per OCM environment to avoid deleting
+                # clusters from other environments
+                vault_input_path=f"{self.params.vault_input_path}/{ocm_env.name}",
+                contacts=self.params.contacts,
+                dry_run=dry_run,
+            )

reconcile/rhidp/sso_client/metrics.py

@@ -0,0 +1,39 @@
+from reconcile.rhidp.metrics import RhIdpBaseMetric
+from reconcile.utils.metrics import (
+    CounterMetric,
+    GaugeMetric,
+)
+
+
+class RhIdpSSOClientReconcileErrorCounter(RhIdpBaseMetric, CounterMetric):
+    """Counter for the failed reconcile runs."""
+
+    @classmethod
+    def name(cls) -> str:
+        return "rhidp_sso_client_reconcile_errors"
+
+
+class RhIdpSSOClientReconcileCounter(RhIdpBaseMetric, CounterMetric):
+    """Counter for successful reconcile runs."""
+
+    @classmethod
+    def name(cls) -> str:
+        return "rhidp_sso_client_reconciled"
+
+
+class RhIdpSSOClientIatExpiration(RhIdpBaseMetric, GaugeMetric):
+    """Gauge for the expiration of the initial access token."""
+
+    path: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "rhidp_sso_client_inital_access_token_expiration"
+
+
+class RhIdpSSOClientCounter(RhIdpBaseMetric, GaugeMetric):
+    """Number of existing SSO clients."""
+
+    @classmethod
+    def name(cls) -> str:
+        return "rhidp_sso_client_number_of_clients"

reconcile/run_integration.py

@@ -0,0 +1,293 @@
+#!/usr/bin/env python3
+
+import logging
+import os
+import sys
+import time
+from collections.abc import Callable
+from importlib import metadata
+
+import click
+from prometheus_client import (
+    push_to_gateway,
+    start_http_server,
+)
+from prometheus_client.exposition import basic_auth_handler
+
+from reconcile.status import ExitCodes
+from reconcile.utils.metrics import (
+    execution_counter,
+    pushgateway_registry,
+    pushgateway_run_status,
+    pushgateway_run_time,
+    run_status,
+    run_time,
+)
+from reconcile.utils.runtime.environment import (
+    LOG_DATEFMT,
+    log_fmt,
+)
+
+SHARDS = int(os.environ.get("SHARDS", 1))
+SHARD_ID = int(os.environ.get("SHARD_ID", 0))
+SHARD_ID_LABEL = os.environ.get("SHARD_KEY", f"{SHARD_ID}-{SHARDS}")
+PREFIX_LOG_LEVEL = os.environ.get("PREFIX_LOG_LEVEL", "false")
+
+INTEGRATION_NAME = os.environ.get("INTEGRATION_NAME")
+COMMAND_NAME = os.environ.get("COMMAND_NAME", "qontract-reconcile")
+
+RUN_ONCE = os.environ.get("RUN_ONCE")
+DRY_RUN = (
+    os.environ.get("MANAGER_DRY_RUN")
+    if INTEGRATION_NAME == "integrations-manager"
+    else os.environ.get("DRY_RUN")
+)
+INTEGRATION_EXTRA_ARGS = os.environ.get("INTEGRATION_EXTRA_ARGS")
+CONFIG = os.environ.get("CONFIG", "/config/config.toml")
+PROMETHEUS_PORT = os.environ.get("PROMETHEUS_PORT", 9090)
+
+LOG_FILE = os.environ.get("LOG_FILE")
+LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO")
+SLEEP_DURATION_SECS = os.environ.get("SLEEP_DURATION_SECS", 600)
+SLEEP_ON_ERROR = os.environ.get("SLEEP_ON_ERROR", 10)
+
+PUSHGATEWAY_ENABLED = os.environ.get("PUSHGATEWAY_ENABLED", False)
+
+LOG = logging.getLogger(__name__)
+
+# Messages to stdout
+STREAM_HANDLER = logging.StreamHandler(sys.stdout)
+STREAM_HANDLER.setFormatter(
+    logging.Formatter(fmt=log_fmt(dry_run_option=DRY_RUN), datefmt=LOG_DATEFMT)
+)
+HANDLERS = [STREAM_HANDLER]
+
+# Messages to the log file
+if LOG_FILE is not None:
+    FILE_HANDLER = logging.FileHandler(LOG_FILE)
+    logFileFormat = "%(message)s"
+    if PREFIX_LOG_LEVEL == "true":
+        logFileFormat = "[%(levelname)s] %(message)s"
+    FILE_HANDLER.setFormatter(logging.Formatter(fmt=logFileFormat))
+    HANDLERS.append(FILE_HANDLER)  # type: ignore
+
+# Setting up the root logger
+logging.basicConfig(level=LOG_LEVEL, handlers=HANDLERS)
+
+
+class PushgatewayBadConfigError(Exception):
+    pass
+
+
+def _parse_dry_run_flag(dry_run: str | None) -> str | None:
+    dry_run_options = ["--dry-run", "--no-dry-run"]
+    if dry_run is not None and dry_run not in dry_run_options:
+        msg = (
+            f'Invalid DRY_RUN option given: "{dry_run}".'
+            f"Only the following options are allowed: {dry_run_options}"
+        )
+        logging.error(msg)
+        raise ValueError(msg)
+    return dry_run if dry_run else None
+
+
+def build_entry_point_args(
+    command: click.Command,
+    config: str,
+    dry_run: str | None,
+    integration_name: str,
+    extra_args: str | None,
+) -> list[str]:
+    args = ["--config", config]
+    if dry_run_flag := _parse_dry_run_flag(dry_run):
+        args.append(dry_run_flag)
+
+    # if the integration_name is a known sub command,
+    # we add it right before the extra_args
+    if (
+        integration_name
+        and isinstance(command, click.MultiCommand)
+        and command.get_command(None, integration_name)  # type: ignore
+    ):
+        args.append(integration_name)
+
+    if extra_args is not None:
+        args.extend(extra_args.split())
+    return args
+
+
+def build_entry_point_func(command_name: str) -> click.Command:
+    """
+    Use the entry point information from setup.py to
+    find the function to invoke for a command.
+    """
+    console_script_entry_points = {
+        ep.name: ep for ep in metadata.entry_points().select(group="console_scripts")
+    }
+    entry_point: metadata.EntryPoint | None = console_script_entry_points.get(
+        command_name
+    )
+    if entry_point:
+        return entry_point.load()
+    raise ValueError(
+        f"Command {command_name} unknown."
+        f"Have a look at setup.py for valid entry points."
+    )
+
+
+def _get_pushgateway_env_vars() -> dict[str, str]:
+    env = {}
+    missing_vars = []
+    for var in ["PUSHGATEWAY_USERNAME", "PUSHGATEWAY_PASSWORD", "PUSHGATEWAY_URL"]:
+        value = os.environ.get(var)
+        if not value:
+            missing_vars.append(var)
+            continue
+
+        env[var] = value
+
+    if missing_vars:
+        missing_str = ", ".join(missing_vars)
+        raise PushgatewayBadConfigError(
+            f"Failed to check env variables to configure Pushgateway: {missing_str}"
+        )
+
+    return env
+
+
+def _push_gateway_basic_auth_handler(
+    url: str,
+    method: str,
+    timeout: float | None,
+    headers: list[tuple[str, str]],
+    data: bytes,
+) -> Callable[[], None]:
+    username = os.environ.get("PUSHGATEWAY_USERNAME")
+    password = os.environ.get("PUSHGATEWAY_PASSWORD")
+
+    # We should not get here, but this will make mypy happy
+    if not username or not password:
+        raise PushgatewayBadConfigError(
+            "Failed to check env variables to configure Pushgateway."
+        )
+
+    return basic_auth_handler(url, method, timeout, headers, data, username, password)
+
+
+def main() -> None:
+    """
+    This entry point script expects certain env variables
+    * COMMAND_NAME (optional, defaults to qontract-reconcile)
+      an entry point as defined in setup.py must be a click.Command
+    * INTEGRATION_NAME
+      used as name for the subcommand for command if present as a subcommand
+      on the click command
+    * INTEGRATION_EXTRA_ARGS (optional)
+      space separated list of arguments that will be passed to the command
+      or subcommand
+    * CONFIG
+      path to the config toml file
+    * LOG_LEVEL
+      Log level (defaults to INFO)
+    * LOG_FILE
+      path for the logfile to write to
+    * DRY_RUN (optional)
+      this is not a boolean but must contain the actual dry-run flag value,
+      so --dry-run or --no-dry-run
+    * RUN_ONCE (optional)
+      if 'true', execute the integration once and exit
+      otherwise run the integration in a loop controlled by SLEEP_DURATION_SECS
+      and SLEEP_ON_ERROR
+    * SLEEP_DURATION_SECS (default 600)
+      amount of seconds to sleep between successful integration runs
+    * SLEEP_ON_ERROR (default 10)
+      amount of seconds to sleep before another integration run is started
+    * PUSHGATEWAY_ENABLED (defaults to false)
+      send metrics to a Prometheus Pushgateway after the run. In expects
+      "PUSHGATEWAY_USERNAME", "PUSHGATEWAY_PASSWORD" and "PUSHGATEWAY_URL" to be defined.
+
+
+    Based on those variables, the following command will be executed
+      $COMMAND --config $CONFIG $DRY_RUN $INTEGRATION_NAME \
+        $INTEGRATION_EXTRA_ARGS
+    """
+    if len(sys.argv) > 1 and sys.argv[1] == "--help":
+        print(main.__doc__)
+        sys.exit(0)
+
+    if not INTEGRATION_NAME:
+        raise ValueError("INTEGRATION_NAME env variable is required")
+
+    start_http_server(int(PROMETHEUS_PORT))
+
+    command = build_entry_point_func(COMMAND_NAME)
+    while True:
+        args = build_entry_point_args(
+            command, CONFIG, DRY_RUN, INTEGRATION_NAME, INTEGRATION_EXTRA_ARGS
+        )
+        sleep = SLEEP_DURATION_SECS
+        start_time = time.monotonic()
+        # Running the integration via Click, so we don't have to replicate
+        # the CLI logic here
+        execution_counter.labels(
+            integration=INTEGRATION_NAME, shards=SHARDS, shard_id=SHARD_ID_LABEL
+        ).inc()
+        try:
+            with command.make_context(info_name=COMMAND_NAME, args=args) as ctx:  # type: ignore
+                ctx.ensure_object(dict)
+                command.invoke(ctx)
+                return_code = 0
+        # This is for when the integration explicitly
+        # calls sys.exit(N)
+        except SystemExit as exc_obj:
+            return_code = int(exc_obj.code)  # type: ignore[arg-type]
+        # We have to be generic since we don't know what can happen
+        # in the integrations, but we want to continue the loop anyway
+        except Exception:
+            sleep = SLEEP_ON_ERROR
+            LOG.exception(f"Error running {COMMAND_NAME}")
+            return_code = ExitCodes.ERROR
+
+        time_spent = time.monotonic() - start_time
+
+        run_time.labels(
+            integration=INTEGRATION_NAME, shards=SHARDS, shard_id=SHARD_ID_LABEL
+        ).set(time_spent)
+        run_status.labels(
+            integration=INTEGRATION_NAME, shards=SHARDS, shard_id=SHARD_ID_LABEL
+        ).set(return_code)
+
+        if PUSHGATEWAY_ENABLED:
+            try:
+                env = _get_pushgateway_env_vars()
+                pushgateway_run_time.labels(
+                    integration=INTEGRATION_NAME, shards=SHARDS, shard_id=SHARD_ID_LABEL
+                ).set(time_spent)
+                pushgateway_run_status.labels(
+                    integration=INTEGRATION_NAME, shards=SHARDS, shard_id=SHARD_ID_LABEL
+                ).set(return_code)
+
+                grouping_key = {
+                    "integration": INTEGRATION_NAME,
+                    "shards": SHARDS,
+                    "shard_id": SHARD_ID_LABEL,
+                }
+                push_to_gateway(
+                    gateway=env["PUSHGATEWAY_URL"],
+                    job="qontract-reconcile",
+                    registry=pushgateway_registry,
+                    handler=_push_gateway_basic_auth_handler,
+                    grouping_key=grouping_key,
+                )
+            except PushgatewayBadConfigError as err:
+                LOG.exception(f"Error pushing to PushGateway: {err}")
+                return_code = ExitCodes.ERROR
+
+        if RUN_ONCE:
+            sys.exit(return_code)
+
+        time.sleep(int(sleep))
+
+
+if __name__ == "__main__":
+    main()