qontract-reconcile 0.10.1rc806__py3-none-any.whl → 0.10.1rc808__py3-none-any.whl

{qontract_reconcile-0.10.1rc806.dist-info → qontract_reconcile-0.10.1rc808.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc806
+Version: 0.10.1rc808
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc806.dist-info → qontract_reconcile-0.10.1rc808.dist-info}/RECORD

@@ -10,7 +10,7 @@ reconcile/aws_iam_password_reset.py,sha256=NwErtrqgBiXr7eGCAHdtGGOx0S7-4JnSc29Ie
 reconcile/aws_support_cases_sos.py,sha256=Jk6_XjDeJSYxgRGqcEAOcynt9qJF2r5HPIPcSKmoBv8,2974
 reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=W_VJagnsJR1v5oqjlI3RJJE0_nhtJ0m81RS8zWA5u5c,3538
 reconcile/checkpoint.py,sha256=R2WFXUXLTB4sWMi4GeA4eegsuf_1-Q4vH8M0Toh3Ij4,5036
-reconcile/cli.py,sha256=pBX9_8GU6g4m8Y3-KeX8wOCv9datyVqxvbkXTmgdvNQ,100833
+reconcile/cli.py,sha256=n64KqkdnqF_H2WHoqlSxQ2PP0uYuY9THPGeyiLP9zJA,101154
 reconcile/closedbox_endpoint_monitoring_base.py,sha256=SMhkcQqprWvThrIJa3U_3uh5w1h-alleW1QnCJFY4Qw,4909
 reconcile/cluster_deployment_mapper.py,sha256=2Ah-nu-Mdig0pjuiZl_XLrmVAjYzFjORR3dMlCgkmw0,2352
 reconcile/dashdotdb_base.py,sha256=a5aPLVxyqPSbjdB0Ty-uliOtxwvEbbEljHJKxdK3-Zk,4813
@@ -104,7 +104,7 @@ reconcile/sendgrid_teammates.py,sha256=oO8QbLb4s1o8A6CGiCagN9CmS05BSS_WLztuY0Ym9
 reconcile/service_dependencies.py,sha256=PMKP9vc6oL-78rzyF_RE8DzLSQMSqN8vCqt9sWpBLAM,4470
 reconcile/signalfx_endpoint_monitoring.py,sha256=D1m8iq0EAKie0OD59FOcVCtpWWZ7xlo6lwBS9urwMIk,2894
 reconcile/slack_base.py,sha256=K3fSYx46G1djoPb07_C9j6ChhMCt5LgV5l6v2TFkNZk,3479
-reconcile/slack_usergroups.py,sha256=sp1QSPRUgTj3-hXpAWx1qOeR6okmVmlHcTUvUMTOLDI,27855
+reconcile/slack_usergroups.py,sha256=vYecHCeJMZhxy1l0bIahTRN6ja0vxTSTXN2UfOkYXOo,29727
 reconcile/sql_query.py,sha256=FAQI9EIHsokZBbGwvGU4vnjg1fHemxpYQE20UtCB1qo,25941
 reconcile/status.py,sha256=cY4IJFXemhxptRJqR4qaaOWqei9e4jgLXuVSGajMsjg,544
 reconcile/status_board.py,sha256=nA74_133jukxVShjPKJpkXOA3vggDTTEhYTegoXbN1M,8632
@@ -794,8 +794,8 @@ tools/saas_metrics_exporter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 tools/saas_metrics_exporter/main.py,sha256=piocx6meMdJxoxeNz52gQGUjt5n7Fma4kgqYamszPrM,3180
 tools/saas_metrics_exporter/commit_distance/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tools/saas_metrics_exporter/commit_distance/channel.py,sha256=XEAh3eL8TmgMe7V2BsyxuXYWgvBBVdSJETd6Ec7cI04,2171
-tools/saas_metrics_exporter/commit_distance/commit_distance.py,sha256=snkcHKS7zxSIomS7psEQ13efN-j9MxKZHe0nLw55dAk,4042
-tools/saas_metrics_exporter/commit_distance/metrics.py,sha256=rQTcinrv3uGLnHFumS37NN3QyVv1z6HGqy8MtfOwcxM,544
+tools/saas_metrics_exporter/commit_distance/commit_distance.py,sha256=nyw9N1ESrcxweWfESI-zlldDdSQGLtiuC6x7DXIjtDw,3074
+tools/saas_metrics_exporter/commit_distance/metrics.py,sha256=5-y6n-sGACAS3eJ5ndY-2BFxcd0fxLfhvZmmBHu4JuA,426
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
 tools/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -804,8 +804,8 @@ tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvf
 tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jrss,4941
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc806.dist-info/METADATA,sha256=n5MHe0IaW3GE0yf5Yy3vSdT-jrs4U3al9qbv4LUPWQ8,2314
-qontract_reconcile-0.10.1rc806.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc806.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc806.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc806.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc808.dist-info/METADATA,sha256=avC3f2n3tg5yBABcDSjLjyLvPd7Axtxg7AYNThYdak4,2314
+qontract_reconcile-0.10.1rc808.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc808.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc808.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc808.dist-info/RECORD,,

reconcile/cli.py

@@ -1200,8 +1200,18 @@ def openshift_upgrade_watcher(ctx, thread_pool_size, internal, use_jump_host):
 @integration.command(short_help="Manage Slack User Groups (channels and users).")
 @workspace_name
 @usergroup_name
+@enable_extended_early_exit
+@extended_early_exit_cache_ttl_seconds
+@log_cached_log_output
 @click.pass_context
-def slack_usergroups(ctx, workspace_name, usergroup_name):
+def slack_usergroups(
+    ctx,
+    workspace_name,
+    usergroup_name,
+    enable_extended_early_exit,
+    extended_early_exit_cache_ttl_seconds,
+    log_cached_log_output,
+):
     import reconcile.slack_usergroups
 
     run_integration(
@@ -1209,6 +1219,9 @@ def slack_usergroups(ctx, workspace_name, usergroup_name):
         ctx.obj,
         workspace_name,
         usergroup_name,
+        enable_extended_early_exit,
+        extended_early_exit_cache_ttl_seconds,
+        log_cached_log_output,
     )
 
 

reconcile/slack_usergroups.py

@@ -9,6 +9,7 @@ from typing import (
     Any,
     Optional,
     Sequence,
+    TypedDict,
     Union,
 )
 from urllib.parse import urlparse
@@ -47,6 +48,10 @@ from reconcile.utils.exceptions import (
     AppInterfaceSettingsError,
     UnknownError,
 )
+from reconcile.utils.extended_early_exit import (
+    ExtendedEarlyExitRunnerResult,
+    extended_early_exit_run,
+)
 from reconcile.utils.github_api import GithubRepositoryApi
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.pagerduty_api import (
@@ -67,6 +72,8 @@ from reconcile.utils.slack_api import (
 
 DATE_FORMAT = "%Y-%m-%d %H:%M"
 QONTRACT_INTEGRATION = "slack-usergroups"
+INTEGRATION_VERSION = "0.1.0"
+
 error_occurred = False
 
 
@@ -511,14 +518,14 @@ def _create_usergroups(
     desired_ug_state: State,
     slack_client: SlackApi,
     dry_run: bool = True,
-) -> None:
+) -> int:
     """Create Slack usergroups."""
     global error_occurred  # noqa: PLW0603
     if current_ug_state:
         logging.debug(
             f"[{desired_ug_state.workspace}] Usergroup exists and will not be created {desired_ug_state.usergroup}"
         )
-        return
+        return 0
 
     logging.info([
         "create_usergroup",
@@ -531,6 +538,7 @@ def _create_usergroups(
         except SlackApiError as error:
             logging.error(error)
             error_occurred = True
+    return 1
 
 
 def _update_usergroup_users_from_state(
@@ -538,14 +546,14 @@ def _update_usergroup_users_from_state(
     desired_ug_state: State,
     slack_client: SlackApi,
     dry_run: bool = True,
-) -> None:
+) -> int:
     """Update the users in a Slack usergroup."""
     global error_occurred  # noqa: PLW0603
     if current_ug_state.user_names == desired_ug_state.user_names:
         logging.debug(
             f"No usergroup user changes detected for {desired_ug_state.usergroup}"
         )
-        return
+        return 0
 
     slack_user_objects = [
         SlackObject(pk=pk, name=name)
@@ -559,7 +567,7 @@ def _update_usergroup_users_from_state(
             f"Following usernames are incorrect for usergroup {desired_ug_state.usergroup} and could not be matched with slack users {desired_ug_state.user_names - set(s.name for s in slack_user_objects)}"
         )
         error_occurred = True
-        return
+        return 0
 
     for user in desired_ug_state.user_names - current_ug_state.user_names:
         logging.info([
@@ -584,7 +592,7 @@ def _update_usergroup_users_from_state(
                 logging.info(
                     f"Usergroup {desired_ug_state.usergroup} does not exist yet. Skipping for now."
                 )
-                return
+                return 0
             slack_client.update_usergroup_users(
                 id=ugid,
                 users_list=sorted([s.pk for s in slack_user_objects]),
@@ -596,6 +604,7 @@ def _update_usergroup_users_from_state(
             # sensitive updates.
             logging.error(error)
             error_occurred = True
+    return 1
 
 
 def _update_usergroup_from_state(
@@ -603,9 +612,10 @@ def _update_usergroup_from_state(
     desired_ug_state: State,
     slack_client: SlackApi,
     dry_run: bool = True,
-) -> None:
+) -> int:
     """Update a Slack usergroup."""
     global error_occurred  # noqa: PLW0603
+    change_detected = False
     if (
         current_ug_state.channel_names == desired_ug_state.channel_names
         and current_ug_state.description == desired_ug_state.description
@@ -613,7 +623,7 @@ def _update_usergroup_from_state(
         logging.debug(
             f"No usergroup channel/description changes detected for {desired_ug_state.usergroup}",
         )
-        return
+        return 0
 
     slack_channel_objects = [
         SlackObject(pk=pk, name=name)
@@ -635,6 +645,7 @@ def _update_usergroup_from_state(
     #     return
 
     for channel in desired_channel_names - current_ug_state.channel_names:
+        change_detected = True
         logging.info([
             "add_channel_to_usergroup",
             desired_ug_state.workspace,
@@ -643,6 +654,7 @@ def _update_usergroup_from_state(
         ])
 
     for channel in current_ug_state.channel_names - desired_channel_names:
+        change_detected = True
         logging.info([
             "del_channel_from_usergroup",
             desired_ug_state.workspace,
@@ -651,6 +663,7 @@ def _update_usergroup_from_state(
         ])
 
     if current_ug_state.description != desired_ug_state.description:
+        change_detected = True
         logging.info([
             "update_usergroup_description",
             desired_ug_state.workspace,
@@ -658,14 +671,14 @@ def _update_usergroup_from_state(
             desired_ug_state.description,
         ])
 
-    if not dry_run:
+    if not dry_run and change_detected:
         try:
             ugid = slack_client.get_usergroup_id(desired_ug_state.usergroup)
             if not ugid:
                 logging.info(
                     f"Usergroup {desired_ug_state.usergroup} does not exist yet. Skipping for now."
                 )
-                return
+                return 0
             slack_client.update_usergroup(
                 id=ugid,
                 channels_list=sorted(s.pk for s in slack_channel_objects),
@@ -674,6 +687,8 @@ def _update_usergroup_from_state(
         except SlackApiError as error:
             logging.error(error)
             error_occurred = True
+        return 1
+    return 0
 
 
 def act(
@@ -681,35 +696,37 @@ def act(
     desired_state: SlackState,
     slack_map: SlackMap,
     dry_run: bool = True,
-) -> None:
+) -> int:
     """Reconcile the differences between the desired and current state for
     Slack usergroups."""
+    apply_count = 0
     for workspace, desired_ws_state in desired_state.items():
         for usergroup, desired_ug_state in desired_ws_state.items():
             current_ug_state: State = current_state.get(workspace, {}).get(
                 usergroup, State()
             )
 
-            _create_usergroups(
+            apply_count += _create_usergroups(
                 current_ug_state,
                 desired_ug_state,
                 slack_client=slack_map[workspace].slack,
                 dry_run=dry_run,
             )
 
-            _update_usergroup_users_from_state(
+            apply_count += _update_usergroup_users_from_state(
                 current_ug_state,
                 desired_ug_state,
                 slack_client=slack_map[workspace].slack,
                 dry_run=dry_run,
             )
 
-            _update_usergroup_from_state(
+            apply_count += _update_usergroup_from_state(
                 current_ug_state,
                 desired_ug_state,
                 slack_client=slack_map[workspace].slack,
                 dry_run=dry_run,
             )
+    return apply_count
 
 
 def get_permissions(query_func: Callable) -> list[PermissionSlackUsergroupV1]:
@@ -731,10 +748,22 @@ def get_clusters(query_func: Callable) -> list[ClusterV1]:
     return clusters_query(query_func=query_func).clusters or []
 
 
+class RunnerParams(TypedDict):
+    dry_run: bool
+    slack_map: SlackMap
+    desired_state: SlackState
+    clusters: list[ClusterV1]
+    workspace_name: Optional[str]
+    usergroup_name: Optional[str]
+
+
 def run(
     dry_run: bool,
     workspace_name: Optional[str] = None,
     usergroup_name: Optional[str] = None,
+    enable_extended_early_exit: bool = False,
+    extended_early_exit_cache_ttl_seconds: int = 3600,
+    log_cached_log_output: bool = False,
 ) -> None:
     global error_occurred  # noqa: PLW0603
     error_occurred = False
@@ -777,6 +806,45 @@ def run(
     # merge the two desired states recursively
     desired_state = deep_update(desired_state, desired_state_cluster_usergroups)
 
+    runner_params: RunnerParams = dict(
+        dry_run=dry_run,
+        slack_map=slack_map,
+        desired_state=desired_state,
+        clusters=clusters,
+        workspace_name=workspace_name,
+        usergroup_name=usergroup_name,
+    )
+
+    if enable_extended_early_exit:
+        extended_early_exit_run(
+            QONTRACT_INTEGRATION,
+            INTEGRATION_VERSION,
+            dry_run,
+            desired_state,
+            "",
+            extended_early_exit_cache_ttl_seconds,
+            logging.getLogger(),
+            runner,
+            runner_params=runner_params,
+            log_cached_log_output=log_cached_log_output,
+            secret_reader=secret_reader,
+        )
+    else:
+        runner(**runner_params)
+
+    if error_occurred:
+        logging.error("Error(s) occurred.")
+        sys.exit(1)
+
+
+def runner(
+    dry_run: bool,
+    slack_map: SlackMap,
+    desired_state: SlackState,
+    clusters: list[ClusterV1],
+    workspace_name: Optional[str] = None,
+    usergroup_name: Optional[str] = None,
+) -> ExtendedEarlyExitRunnerResult:
     current_state = get_current_state(
         slack_map=slack_map,
         desired_workspace_name=workspace_name,
@@ -787,15 +855,13 @@ def run(
             if integration_is_enabled(QONTRACT_INTEGRATION, cluster)
         ],
     )
-    act(
+    apply_count = act(
         current_state=current_state,
         desired_state=desired_state,
         slack_map=slack_map,
         dry_run=dry_run,
     )
-    if error_occurred:
-        logging.error("Error(s) occurred.")
-        sys.exit(1)
+    return ExtendedEarlyExitRunnerResult(payload={}, applied_count=apply_count)
 
 
 def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:

tools/saas_metrics_exporter/commit_distance/commit_distance.py

@@ -7,124 +7,97 @@ from reconcile.typed_queries.saas_files import SaasFile
 from reconcile.utils.secret_reader import HasSecret
 from reconcile.utils.vcs import VCS
 from tools.saas_metrics_exporter.commit_distance.channel import (
-    Channel,
     SaasTarget,
     build_channels,
 )
 from tools.saas_metrics_exporter.commit_distance.metrics import SaasCommitDistanceGauge
 
 
-@dataclass
-class Distance:
-    publisher: SaasTarget
-    subscriber: SaasTarget
-    channel: Channel
-    distance: int = 0
-
-
 @dataclass
 class CommitDistanceMetric:
     value: float
     metric: SaasCommitDistanceGauge
 
 
-@dataclass
-class ThreadData:
+@dataclass(frozen=True)
+class DistanceKey:
     repo_url: str
     auth_code: HasSecret | None
     ref_from: str
     ref_to: str
-    distance: int = 0
-
-    def __hash__(self) -> int:
-        return hash((self.repo_url, self.ref_from, self.ref_to))
 
 
 class CommitDistanceFetcher:
     def __init__(self, vcs: VCS):
         self._vcs = vcs
 
-    def _data_key(self, repo_url: str, ref_from: str, ref_to: str) -> str:
-        return f"{repo_url}/{ref_from}/{ref_to}"
-
-    def _calculate_commit_distance(self, data: ThreadData) -> None:
-        if data.ref_from == data.ref_to:
-            data.distance = 0
-            return
+    def _calculate_commit_distance(self, key: DistanceKey) -> tuple[DistanceKey, int]:
+        if key.ref_from == key.ref_to:
+            return key, 0
 
         commits = self._vcs.get_commits_between(
-            repo_url=data.repo_url,
-            auth_code=data.auth_code,
-            commit_from=data.ref_from,
-            commit_to=data.ref_to,
+            repo_url=key.repo_url,
+            auth_code=key.auth_code,
+            commit_from=key.ref_from,
+            commit_to=key.ref_to,
+        )
+        return key, len(commits)
+
+    @staticmethod
+    def _build_distance_key(
+        publisher: SaasTarget, subscriber: SaasTarget
+    ) -> DistanceKey:
+        return DistanceKey(
+            repo_url=publisher.repo_url,
+            auth_code=publisher.auth_code,
+            ref_from=subscriber.ref,
+            ref_to=publisher.ref,
         )
-        data.distance = len(commits)
-
-    def _populate_distances(
-        self, distances: Iterable[Distance], thread_data: Iterable[ThreadData]
-    ) -> None:
-        m = {
-            self._data_key(
-                repo_url=d.repo_url, ref_from=d.ref_from, ref_to=d.ref_to
-            ): d.distance
-            for d in thread_data
-        }
-        for distance in distances:
-            distance.distance = m[
-                self._data_key(
-                    repo_url=distance.publisher.repo_url,
-                    ref_from=distance.subscriber.ref,
-                    ref_to=distance.publisher.ref,
-                )
-            ]
 
     def fetch(
-        self, saas_files: Iterable[SaasFile], thread_pool_size: int
+        self,
+        saas_files: Iterable[SaasFile],
+        thread_pool_size: int,
     ) -> list[CommitDistanceMetric]:
         channels = build_channels(saas_files=saas_files)
-        distances: list[Distance] = []
-        thread_data: set[ThreadData] = set()
-
-        for channel in channels:
-            for subscriber in channel.subscribers:
-                for publisher in channel.publishers:
-                    thread_data.add(
-                        ThreadData(
-                            repo_url=publisher.repo_url,
-                            auth_code=publisher.auth_code,
-                            ref_from=subscriber.ref,
-                            ref_to=publisher.ref,
-                        )
-                    )
-                    distances.append(
-                        Distance(
-                            publisher=publisher,
-                            subscriber=subscriber,
-                            channel=channel,
-                        )
-                    )
 
-        threaded.run(
-            self._calculate_commit_distance,
-            thread_data,
-            thread_pool_size=thread_pool_size,
-        )
+        distance_keys = {
+            self._build_distance_key(publisher=publisher, subscriber=subscriber)
+            for channel in channels
+            for subscriber in channel.subscribers
+            for publisher in channel.publishers
+        }
 
-        self._populate_distances(distances=distances, thread_data=thread_data)
+        distance_by_key = dict(
+            threaded.run(
+                self._calculate_commit_distance,
+                distance_keys,
+                thread_pool_size=thread_pool_size,
+            )
+        )
 
         commit_distance_metrics = [
             CommitDistanceMetric(
-                value=float(distance.distance),
+                value=float(
+                    distance_by_key[
+                        self._build_distance_key(
+                            publisher=publisher,
+                            subscriber=subscriber,
+                        )
+                    ]
+                ),
                 metric=SaasCommitDistanceGauge(
-                    channel=distance.channel.name,
-                    app=distance.publisher.app_name,
-                    publisher=distance.publisher.target_name,
-                    publisher_namespace=distance.publisher.namespace_name,
-                    subscriber=distance.subscriber.target_name,
-                    subscriber_namespace=distance.subscriber.namespace_name,
+                    channel=channel.name,
+                    app=publisher.app_name,
+                    publisher=publisher.target_name,
+                    publisher_namespace=publisher.namespace_name,
+                    subscriber=subscriber.target_name,
+                    subscriber_namespace=subscriber.namespace_name,
                 ),
             )
-            for distance in distances
+            for channel in channels
+            for subscriber in channel.subscribers
+            for publisher in channel.publishers
         ]
 
         return commit_distance_metrics

tools/saas_metrics_exporter/commit_distance/metrics.py

@@ -1,19 +1,12 @@
-from pydantic import BaseModel
-
 from reconcile.utils.metrics import (
     GaugeMetric,
 )
 
 
-class SaasBaseMetric(BaseModel):
-    "Base class for Saas metrics"
-
-    integration: str = "saas_metrics_exporter"
-
-
-class SaasCommitDistanceGauge(SaasBaseMetric, GaugeMetric):
+class SaasCommitDistanceGauge(GaugeMetric):
     "Gauge for the commit distance between saas targets in a channel"
 
+    integration: str = "saas_metrics_exporter"
     channel: str
     publisher: str
     publisher_namespace: str