qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/oum/standalone.py

@@ -0,0 +1,196 @@
+import logging
+from collections import defaultdict
+from datetime import timedelta
+
+from reconcile.gql_definitions.fragments.ocm_environment import OCMEnvironment
+from reconcile.oum.base import OCMUserManagementIntegration
+from reconcile.oum.labelset import build_cluster_config_from_labels
+from reconcile.oum.models import (
+    ClusterError,
+    ClusterUserManagementConfiguration,
+    ClusterUserManagementSpec,
+    OrganizationUserManagementConfiguration,
+)
+from reconcile.utils.ocm.base import (
+    OCMClusterServiceLogCreateModel,
+    OCMServiceLogSeverity,
+)
+from reconcile.utils.ocm.clusters import (
+    ClusterDetails,
+    discover_clusters_by_labels,
+)
+from reconcile.utils.ocm.labels import build_container_for_prefix
+from reconcile.utils.ocm.search_filters import Filter
+from reconcile.utils.ocm.service_log import create_service_log
+from reconcile.utils.ocm.sre_capability_labels import sre_capability_label_key
+from reconcile.utils.ocm_base_client import (
+    OCMBaseClient,
+    init_ocm_base_client,
+)
+
+
+class OCMStandaloneUserManagementIntegration(OCMUserManagementIntegration):
+    @property
+    def name(self) -> str:
+        return "ocm-standalone-user-management"
+
+    def get_user_mgmt_config_for_ocm_env(
+        self, ocm_env: OCMEnvironment, org_ids: set[str] | None
+    ) -> dict[str, OrganizationUserManagementConfiguration]:
+        ocm_api = init_ocm_base_client(ocm_env, self.secret_reader)
+        clusters_by_org = discover_clusters(
+            ocm_api=ocm_api,
+            org_ids=org_ids,
+        )
+        configs: dict[str, OrganizationUserManagementConfiguration] = {}
+        for org_id, org_clusters in clusters_by_org.items():
+            configs[org_id] = build_user_management_configurations(
+                org_id=org_id,
+                clusters=org_clusters,
+                providers=self.group_member_provider_ids,
+            )
+        return configs
+
+    def signal_cluster_reconcile_success(
+        self,
+        dry_run: bool,
+        ocm_api: OCMBaseClient,
+        spec: ClusterUserManagementSpec,
+        message: str,
+    ) -> None:
+        logging.info(message)
+        if dry_run:
+            return
+        create_service_log(
+            ocm_api=ocm_api,
+            service_log=OCMClusterServiceLogCreateModel(
+                cluster_uuid=spec.cluster.ocm_cluster.external_id,
+                severity=OCMServiceLogSeverity.Info,
+                summary="Reconciled cluster groups",
+                description=message,
+                service_name=self.name,
+            ),
+        )
+
+    def signal_cluster_validation_error(
+        self,
+        dry_run: bool,
+        ocm_api: OCMBaseClient,
+        spec: ClusterUserManagementSpec,
+        error: Exception,
+    ) -> None:
+        """
+        The standalone user management capability will not fail on a configuration
+        validation issue. If issues should be noticed by an SRE team, alerts based on
+        the `oum_organization_validation_errors` metric in the `reconcile.oum.metrics`
+        module should be set up.
+        """
+        logging.warning(
+            "Failed to reconcile cluster user group configuration in "
+            f"OCM organization {spec.cluster.organization_id}",
+            exc_info=error,
+        )
+        if dry_run:
+            return
+        create_service_log(
+            ocm_api=ocm_api,
+            service_log=OCMClusterServiceLogCreateModel(
+                cluster_uuid=spec.cluster.ocm_cluster.external_id,
+                severity=OCMServiceLogSeverity.Error,
+                summary="Failed to reconcile cluster user group configuration",
+                description=str(error),
+                service_name=self.name,
+            ),
+            dedup_interval=timedelta(days=2),
+        )
+
+    def signal_cluster_reconcile_error(
+        self,
+        dry_run: bool,
+        ocm_api: OCMBaseClient,
+        spec: ClusterUserManagementSpec,
+        error: Exception,
+    ) -> None:
+        """
+        The standalone user management capability will not fail on a cluster
+        reconciliation issue. If issues should be noticed by an SRE team, alerts based on
+        the `oum_organization_reconcile_errors` metric in the `reconcile.oum.metrics`
+        module should be set up.
+
+        The user is not notified via service logs because such reconcile issues are not
+        actionable to them.
+        """
+        logging.warning(
+            "Failed to reconcile cluster user group configuration on "
+            f"cluster {spec.cluster.ocm_cluster.name} (id={spec.cluster.ocm_cluster.id}) "
+            f"in the OCM organization {spec.cluster.organization_id}",
+            exc_info=error,
+        )
+
+
+def user_mgmt_label_key(config_atom: str) -> str:
+    """
+    Generates label keys for the user management authz capability, compliant with the naming
+    scheme defined in https://service.pages.redhat.com/dev-guidelines/docs/sre-capabilities/framework/ocm-labels/
+    """
+    return sre_capability_label_key("user-mgmt", config_atom)
+
+
+def discover_clusters(
+    ocm_api: OCMBaseClient,
+    org_ids: set[str] | None = None,
+) -> dict[str, list[ClusterDetails]]:
+    """
+    Discover all clusters with user management enabled on their subscription
+    or their organization. Return the discovered clusters grouped by OCM organization ID.
+
+    If `org_ids` is provided, only clusters from the specified organizations will be returned.
+    """
+    clusters = discover_clusters_by_labels(
+        ocm_api=ocm_api,
+        label_filter=Filter().like("key", user_mgmt_label_key("%")),
+    )
+
+    # group by org ID
+    # optionally also filter on org IDs
+    clusters_by_org: dict[str, list[ClusterDetails]] = defaultdict(list)
+    for c in clusters:
+        passed_ocm_filters = org_ids is None or c.organization_id in org_ids
+        if passed_ocm_filters:
+            clusters_by_org[c.organization_id].append(c)
+
+    return clusters_by_org
+
+
+def build_user_management_configurations(
+    org_id: str,
+    clusters: list[ClusterDetails],
+    providers: set[str],
+) -> OrganizationUserManagementConfiguration:
+    """
+    Extracts the user management configuration from the cluster labels.
+    """
+    org_config = OrganizationUserManagementConfiguration(org_id=org_id)
+    for c in clusters:
+        cluster_config = ClusterUserManagementConfiguration(cluster=c)
+        org_config.cluster_configs.append(cluster_config)
+        for p in providers:
+            provider_org_labels = build_container_for_prefix(
+                c.organization_labels, user_mgmt_label_key(f"{p}."), True
+            )
+            provider_subs_labels = build_container_for_prefix(
+                c.subscription_labels, user_mgmt_label_key(f"{p}."), True
+            )
+            if provider_org_labels or provider_subs_labels:
+                try:
+                    role_groups = build_cluster_config_from_labels(
+                        provider=p,
+                        org_labels=provider_org_labels,
+                        subscription_labels=provider_subs_labels,
+                    )
+                    for role_id, group_refs in role_groups.items():
+                        cluster_config.roles[role_id].extend(group_refs)
+
+                except Exception as e:
+                    cluster_config.errors.append(ClusterError(message=str(e)))
+    return org_config

reconcile/prometheus_rules_tester/integration.py

@@ -1,16 +1,15 @@
 import json
 import logging
 import sys
+from collections import defaultdict
 from collections.abc import (
     Iterable,
     Mapping,
 )
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 import yaml
+from deepdiff import DeepHash
 from pydantic import BaseModel
 from sretoolbox.utils import threaded
 
@@ -27,6 +26,7 @@ from reconcile.utils import (
     gql,
     promtool,
 )
+from reconcile.utils.jinja2.utils import process_extracurlyjinja2_template
 from reconcile.utils.runtime.integration import DesiredStateShardConfig
 from reconcile.utils.semver_helper import make_semver
 from reconcile.utils.structs import CommandExecutionResult
@@ -55,8 +55,8 @@ class Test(BaseModel):
     rule_path: str
     rule: dict
     rule_length: int
-    tests: Optional[list[TestContent]]
-    result: Optional[CommandExecutionResult] = None
+    tests: list[TestContent] | None
+    result: CommandExecutionResult | None = None
 
 
 class RuleToFetch(BaseModel):
@@ -89,7 +89,7 @@ def fetch_rule_and_tests(
         test_raw_yaml = gql.get_resource(test_path)["content"]
 
         if rule.resource["type"] == "resource-template-extracurlyjinja2":
-            test_raw_yaml = orb.process_extracurlyjinja2_template(
+            test_raw_yaml = process_extracurlyjinja2_template(
                 body=test_raw_yaml,
                 vars=variables,
                 settings=vault_settings.dict(by_alias=True),
@@ -119,18 +119,21 @@ def fetch_rule_and_tests(
 def get_rules_and_tests(
     vault_settings: AppInterfaceSettingsV1,
     thread_pool_size: int,
-    cluster_name: Optional[str] = None,
+    cluster_names: Iterable[str] | None = None,
 ) -> list[Test]:
     """Iterates through all namespaces and returns a list of tests to run"""
     namespace_with_prom_rules, _ = orb.get_namespaces(
         PROVIDERS,
-        cluster_name=cluster_name,
+        cluster_names=cluster_names if cluster_names else [],
         namespace_name=NAMESPACE_NAME,
     )
 
     iterable = []
     for namespace in namespace_with_prom_rules:
-        for resource in namespace["openshiftResources"]:
+        prom_rules = [
+            r for r in namespace["openshiftResources"] if r["provider"] in PROVIDERS
+        ]
+        for resource in prom_rules:
             iterable.append(
                 RuleToFetch(
                     namespace=namespace,
@@ -219,14 +222,14 @@ def check_rules_and_tests(
     vault_settings: AppInterfaceSettingsV1,
     alerting_services: Iterable[str],
     thread_pool_size: int,
-    cluster_name: Optional[str] = None,
+    cluster_names: Iterable[str] | None = None,
 ) -> list[Test]:
     """Fetch rules and associated tests, run checks on rules and tests if they exist
     and return a list of failed checks/tests"""
     tests = get_rules_and_tests(
         vault_settings=vault_settings,
         thread_pool_size=thread_pool_size,
-        cluster_name=cluster_name,
+        cluster_names=cluster_names,
     )
     threaded.run(
         func=run_test,
@@ -241,14 +244,14 @@ def check_rules_and_tests(
 
 
 def run(
-    dry_run: bool, thread_pool_size: int, cluster_name: Optional[str] = None
+    dry_run: bool, thread_pool_size: int, cluster_names: Iterable[str] | None = None
 ) -> None:
     """Check prometheus rules syntax and run the tests associated to them"""
     orb.QONTRACT_INTEGRATION = QONTRACT_INTEGRATION
     orb.QONTRACT_INTEGRATION_VERSION = QONTRACT_INTEGRATION_VERSION
 
     failed_tests = check_rules_and_tests(
-        cluster_name=cluster_name,
+        cluster_names=cluster_names,
         vault_settings=get_app_interface_vault_settings(),
         alerting_services=get_alerting_services(),
         thread_pool_size=thread_pool_size,
@@ -266,14 +269,19 @@ def run(
 def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:
     # early_exit doesn't support dataclasses or BaseModels yet, hence we have to store dicts
     with orb.early_exit_monkey_patch():
+        state_for_clusters = defaultdict(list)
+        tests = get_rules_and_tests(
+            vault_settings=get_app_interface_vault_settings(),
+            thread_pool_size=10,
+        )
+        for t in tests:
+            state_for_clusters[t.cluster_name].append(t)
+
         state = {
-            "tests": [
-                t.dict()
-                for t in get_rules_and_tests(
-                    vault_settings=get_app_interface_vault_settings(),
-                    thread_pool_size=10,
-                )
-            ]
+            "state": {
+                cluster: {"shard": cluster, "hash": DeepHash(state).get(state)}
+                for cluster, state in state_for_clusters.items()
+            }
         }
 
     return state
@@ -281,7 +289,7 @@ def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:
 
 def desired_state_shard_config() -> DesiredStateShardConfig:
     return DesiredStateShardConfig(
-        shard_arg_name="cluster_name",
-        shard_path_selectors={"tests[*].cluster_name"},
+        shard_arg_name="cluster_names",
+        shard_path_selectors={"state.*.shard"},
         sharded_run_review=lambda proposal: len(proposal.proposed_shards) <= 4,
     )

reconcile/quay_base.py

@@ -22,7 +22,10 @@ def get_quay_api_store():
         org_name = org_data["name"]
         org_key = OrgKey(instance_name, org_name)
         base_url = org_data["instance"]["url"]
-        token = secret_reader.read(org_data["automationToken"])
+        automation_token = org_data["automationToken"]
+        if not automation_token:
+            continue
+        token = secret_reader.read(automation_token)
 
         if org_data.get("mirror"):
             mirror = OrgKey(

reconcile/quay_membership.py

@@ -1,7 +1,6 @@
 import logging
 import sys
 from collections.abc import Sequence
-from typing import Union
 
 from reconcile.gql_definitions.quay_membership import quay_membership
 from reconcile.gql_definitions.quay_membership.quay_membership import (
@@ -81,7 +80,7 @@ def fetch_current_state(quay_api_store):
     return state
 
 
-def get_usernames(users: Sequence[Union[UserV1, BotV1]]) -> list[str]:
+def get_usernames(users: Sequence[UserV1 | BotV1]) -> list[str]:
     return [u.quay_username for u in users if u.quay_username]
 
 

reconcile/quay_mirror.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import re
 import sys
 import tempfile
 import time
@@ -11,9 +10,10 @@ from collections import (
 from collections.abc import Iterable
 from typing import (
     Any,
-    Optional,
+    Self,
 )
 
+import requests
 from requests import Response
 from sretoolbox.container.image import (
     ImageComparisonError,
@@ -28,6 +28,7 @@ from reconcile.utils import (
     metrics,
     sharding,
 )
+from reconcile.utils.helpers import match_patterns
 from reconcile.utils.instrumented_wrappers import InstrumentedImage as Image
 from reconcile.utils.instrumented_wrappers import InstrumentedSkopeo as Skopeo
 from reconcile.utils.secret_reader import SecretReader
@@ -36,12 +37,12 @@ _LOG = logging.getLogger(__name__)
 
 QONTRACT_INTEGRATION = "quay-mirror"
 CONTROL_FILE_NAME = "qontract-reconcile-quay-mirror.timestamp"
+REQUEST_TIMEOUT = 60
 
 OrgKey = namedtuple("OrgKey", ["instance", "org_name"])
 
 
 class QuayMirror:
-
     QUAY_ORG_CATALOG_QUERY = """
     {
       quay_orgs: quay_orgs_v1 {
@@ -65,10 +66,11 @@ class QuayMirror:
     def __init__(
         self,
         dry_run: bool = False,
-        control_file_dir: Optional[str] = None,
-        compare_tags: Optional[bool] = None,
+        control_file_dir: str | None = None,
+        compare_tags: bool | None = None,
         compare_tags_interval: int = 86400,
-        images: Optional[Iterable[str]] = None,
+        repository_urls: Iterable[str] | None = None,
+        exclude_repository_urls: Iterable[str] | None = None,
     ) -> None:
         self.dry_run = dry_run
         self.gqlapi = gql.get_api()
@@ -78,7 +80,8 @@ class QuayMirror:
         self.push_creds = self._get_push_creds()
         self.compare_tags = compare_tags
         self.compare_tags_interval = compare_tags_interval
-        self.images = images
+        self.repository_urls = repository_urls
+        self.exclude_repository_urls = exclude_repository_urls
 
         self.response_cache_hits = metrics.cache_hits.labels(
             integration=QONTRACT_INTEGRATION,
@@ -109,7 +112,14 @@ class QuayMirror:
                 tempfile.gettempdir(), CONTROL_FILE_NAME
             )
 
-        self._has_enough_time_passed_since_last_compare_tags: Optional[bool] = None
+        self._has_enough_time_passed_since_last_compare_tags: bool | None = None
+        self.session = requests.Session()
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        self.session.close()
 
     def run(self) -> None:
         sync_tasks = self.process_sync_tasks()
@@ -130,10 +140,20 @@ class QuayMirror:
 
     @classmethod
     def process_repos_query(
-        cls, images: Optional[Iterable[str]] = None
+        cls,
+        repository_urls: Iterable[str] | None = None,
+        exclude_repository_urls: Iterable[str] | None = None,
+        session: requests.Session | None = None,
+        timeout: int = REQUEST_TIMEOUT,
     ) -> defaultdict[OrgKey, list[dict[str, Any]]]:
         apps = queries.get_quay_repos()
 
+        if repository_urls:
+            repository_urls = set(repository_urls)
+
+        if exclude_repository_urls:
+            exclude_repository_urls = set(exclude_repository_urls)
+
         summary = defaultdict(list)
 
         for app in apps:
@@ -148,14 +168,25 @@ class QuayMirror:
                 server_url = quay_repo["org"]["instance"]["url"]
 
                 for item in quay_repo["items"]:
-                    if images and item["name"] not in images:
+                    if item["mirror"] is None:
                         continue
 
-                    if item["mirror"] is None:
+                    mirror_url = item["mirror"]["url"]
+
+                    if repository_urls and mirror_url not in repository_urls:
+                        continue
+
+                    if (
+                        exclude_repository_urls
+                        and mirror_url in exclude_repository_urls
+                    ):
                         continue
 
                     mirror_image = Image(
-                        item["mirror"]["url"], response_cache=cls.response_cache
+                        mirror_url,
+                        response_cache=cls.response_cache,
+                        session=session,
+                        timeout=timeout,
                     )
                     if mirror_image.registry == "docker.io" and item["public"]:
                         _LOG.error(
@@ -166,39 +197,53 @@ class QuayMirror:
                         sys.exit(ExitCodes.ERROR)
 
                     org_key = OrgKey(instance, org)
-                    summary[org_key].append(
-                        {
-                            "name": item["name"],
-                            "mirror": item["mirror"],
-                            "server_url": server_url,
-                        }
-                    )
+                    summary[org_key].append({
+                        "name": item["name"],
+                        "mirror": item["mirror"],
+                        "server_url": server_url,
+                    })
         return summary
 
     @staticmethod
-    def sync_tag(tags, tags_exclude, candidate):
-        if tags is not None:
-            for tag in tags:
-                if re.match(tag, candidate):
-                    return True
-            # When tags is defined, we don't look at
-            # tags_exclude
-            return False
-
-        if tags_exclude is not None:
-            for tag_exclude in tags_exclude:
-                if re.match(tag_exclude, candidate):
-                    return False
+    def sync_tag(
+        tags: Iterable[str] | None,
+        tags_exclude: Iterable[str] | None,
+        candidate: str,
+    ) -> bool:
+        """
+        Determine if the candidate tag should sync, tags_exclude check take precedence.
+        :param tags: regex patterns to filter, match means to sync, None means no filter
+        :param tags_exclude: regex patterns to filter, match means not to sync, None means no filter
+        :param candidate: tag to check
+        :return: bool, True means to sync, False means not to sync
+        """
+        if not tags and not tags_exclude:
             return True
 
-        # Both tags and tags_exclude are None, so
-        # tag must be synced
-        return True
+        if not tags:
+            # only tags_exclude provided
+            assert tags_exclude  # mypy can't infer not None
+            return not match_patterns(tags_exclude, candidate)
+
+        if not tags_exclude:
+            # only tags provided
+            return match_patterns(tags, candidate)
+
+        # both tags and tags_exclude provided
+        return not match_patterns(
+            tags_exclude,
+            candidate,
+        ) and match_patterns(
+            tags,
+            candidate,
+        )
 
     def process_sync_tasks(self):
         if self.is_compare_tags:
             _LOG.warning("Making a compare-tags run. This is a slow operation.")
-        summary = self.process_repos_query(self.images)
+        summary = self.process_repos_query(
+            self.repository_urls, self.exclude_repository_urls
+        )
         sync_tasks = defaultdict(list)
         for org_key, data in summary.items():
             org = org_key.org_name
@@ -209,6 +254,8 @@ class QuayMirror:
                     username=push_creds[0],
                     password=push_creds[1],
                     response_cache=self.response_cache,
+                    session=self.session,
+                    timeout=REQUEST_TIMEOUT,
                 )
 
                 mirror_url = item["mirror"]["url"]
@@ -228,6 +275,8 @@ class QuayMirror:
                     username=username,
                     password=password,
                     response_cache=self.response_cache,
+                    session=self.session,
+                    timeout=REQUEST_TIMEOUT,
                 )
 
                 tags = item["mirror"].get("tags")
@@ -305,13 +354,11 @@ class QuayMirror:
                     _LOG.debug(
                         "Image %s and mirror %s are out of sync", downstream, upstream
                     )
-                    sync_tasks[org_key].append(
-                        {
-                            "mirror_url": str(upstream),
-                            "mirror_creds": mirror_creds,
-                            "image_url": str(downstream),
-                        }
-                    )
+                    sync_tasks[org_key].append({
+                        "mirror_url": str(upstream),
+                        "mirror_creds": mirror_creds,
+                        "image_url": str(downstream),
+                    })
 
         return sync_tasks
 
@@ -336,20 +383,17 @@ class QuayMirror:
     @staticmethod
     def check_compare_tags_elapsed_time(path, interval) -> bool:
         try:
-            with open(path, "r") as file_obj:
+            with open(path, encoding="locale") as file_obj:
                 last_compare_tags = float(file_obj.read())
         except FileNotFoundError:
             return True
 
         next_compare_tags = last_compare_tags + interval
-        if time.time() >= next_compare_tags:
-            return True
-
-        return False
+        return time.time() >= next_compare_tags
 
     @staticmethod
     def record_timestamp(path) -> None:
-        with open(path, "w") as file_object:
+        with open(path, "w", encoding="locale") as file_object:
             file_object.write(str(time.time()))
 
     def _get_push_creds(self):
@@ -372,20 +416,26 @@ class QuayMirror:
 
 def run(
     dry_run,
-    control_file_dir: Optional[str],
-    compare_tags: Optional[bool],
+    control_file_dir: str | None,
+    compare_tags: bool | None,
     compare_tags_interval: int,
-    images: Optional[Iterable[str]],
+    repository_urls: Iterable[str] | None,
+    exclude_repository_urls: Iterable[str] | None,
 ):
-    quay_mirror = QuayMirror(
-        dry_run, control_file_dir, compare_tags, compare_tags_interval, images
-    )
-    quay_mirror.run()
+    with QuayMirror(
+        dry_run,
+        control_file_dir,
+        compare_tags,
+        compare_tags_interval,
+        repository_urls,
+        exclude_repository_urls,
+    ) as quay_mirror:
+        quay_mirror.run()
 
 
 def early_exit_desired_state(*args, **kwargs) -> dict[str, Any]:
-    quay_mirror = QuayMirror(dry_run=True)
-    return {
-        "repos": quay_mirror.process_repos_query(),
-        "orgs": quay_mirror.push_creds,
-    }
+    with QuayMirror(dry_run=True) as quay_mirror:
+        return {
+            "repos": quay_mirror.process_repos_query(session=quay_mirror.session),
+            "orgs": quay_mirror.push_creds,
+        }