qontract-reconcile 0.10.1rc805__py3-none-any.whl → 0.10.1rc806__py3-none-any.whl

{qontract_reconcile-0.10.1rc805.dist-info → qontract_reconcile-0.10.1rc806.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc805
+Version: 0.10.1rc806
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc805.dist-info → qontract_reconcile-0.10.1rc806.dist-info}/RECORD

@@ -790,15 +790,22 @@ tools/cli_commands/cost_report/openshift.py,sha256=XNEJpgtIQAi3Eoej1Btnl74IWg3g-
 tools/cli_commands/cost_report/response.py,sha256=_kbpBSjMjbPXGkjDgidTOLvFpVqfBo3VMkSOheUdmMA,1308
 tools/cli_commands/cost_report/util.py,sha256=r4K8nC1S0YZNSUNro141cYG1nuG8HwkkYEqWV9GCvu8,1861
 tools/cli_commands/cost_report/view.py,sha256=pKM6LDeWZJcTw2a7sWBwKSuR9p3SAk3lEb37uwMhlLw,10183
+tools/saas_metrics_exporter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tools/saas_metrics_exporter/main.py,sha256=piocx6meMdJxoxeNz52gQGUjt5n7Fma4kgqYamszPrM,3180
+tools/saas_metrics_exporter/commit_distance/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tools/saas_metrics_exporter/commit_distance/channel.py,sha256=XEAh3eL8TmgMe7V2BsyxuXYWgvBBVdSJETd6Ec7cI04,2171
+tools/saas_metrics_exporter/commit_distance/commit_distance.py,sha256=snkcHKS7zxSIomS7psEQ13efN-j9MxKZHe0nLw55dAk,4042
+tools/saas_metrics_exporter/commit_distance/metrics.py,sha256=rQTcinrv3uGLnHFumS37NN3QyVv1z6HGqy8MtfOwcxM,544
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
 tools/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tools/test/conftest.py,sha256=BYS85m3oTUxf7gxku14oH0ctlPMS1x1nk69kEJ6G5cc,1022
 tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvftCWEEf-g1mfXOtgCog-g,1271
 tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jrss,4941
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc805.dist-info/METADATA,sha256=hQqEKCmIWG-75Lp4rO5VuV1Utx4ciyWOVz3hcotymnM,2314
-qontract_reconcile-0.10.1rc805.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc805.dist-info/entry_points.txt,sha256=rIxI5zWtHNlfpDeq1a7pZXAPoqf7HG32KMTN3MeWK_8,429
-qontract_reconcile-0.10.1rc805.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc805.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc806.dist-info/METADATA,sha256=n5MHe0IaW3GE0yf5Yy3vSdT-jrs4U3al9qbv4LUPWQ8,2314
+qontract_reconcile-0.10.1rc806.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc806.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc806.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc806.dist-info/RECORD,,

{qontract_reconcile-0.10.1rc805.dist-info → qontract_reconcile-0.10.1rc806.dist-info}/entry_points.txt

@@ -5,4 +5,5 @@ glitchtip-access-reporter = tools.glitchtip_access_reporter:main
 glitchtip-access-revalidation = tools.glitchtip_access_revalidation:main
 qontract-cli = tools.qontract_cli:root
 qontract-reconcile = reconcile.cli:integration
+saas-metrics-exporter = tools.saas_metrics_exporter.main:main
 template-validation = tools.template_validation:main

tools/saas_metrics_exporter/commit_distance/channel.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+from reconcile.typed_queries.saas_files import SaasFile
+from reconcile.utils.secret_reader import HasSecret
+
+
+@dataclass
+class SaasTarget:
+    app_name: str
+    repo_url: str
+    namespace_name: str
+    target_name: str
+    ref: str
+    auth_code: HasSecret | None
+
+
+@dataclass
+class Channel:
+    name: str
+    subscribers: list[SaasTarget]
+    publishers: list[SaasTarget]
+
+
+def build_channels(saas_files: Iterable[SaasFile]) -> list[Channel]:
+    channels: dict[str, Channel] = {}
+    for saas_file in saas_files:
+        for resource_template in saas_file.resource_templates:
+            for target in resource_template.targets:
+                if not target.promotion:
+                    continue
+                if not (target.promotion.publish or target.promotion.subscribe):
+                    continue
+                auth_code = (
+                    saas_file.authentication.code if saas_file.authentication else None
+                )
+                target_name = target.name if target.name else "NoName"
+                saas_target = SaasTarget(
+                    app_name=saas_file.app.name,
+                    repo_url=resource_template.url,
+                    ref=target.ref,
+                    auth_code=auth_code,
+                    namespace_name=target.namespace.name,
+                    target_name=target_name,
+                )
+
+                for channel in target.promotion.publish or []:
+                    if channel not in channels:
+                        channels[channel] = Channel(
+                            name=channel, subscribers=[], publishers=[]
+                        )
+                    channels[channel].publishers.append(saas_target)
+
+                for channel in target.promotion.subscribe or []:
+                    if channel not in channels:
+                        channels[channel] = Channel(
+                            name=channel, subscribers=[], publishers=[]
+                        )
+                    channels[channel].subscribers.append(saas_target)
+
+    return list(channels.values())

tools/saas_metrics_exporter/commit_distance/commit_distance.py

@@ -0,0 +1,130 @@
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+from sretoolbox.utils import threaded
+
+from reconcile.typed_queries.saas_files import SaasFile
+from reconcile.utils.secret_reader import HasSecret
+from reconcile.utils.vcs import VCS
+from tools.saas_metrics_exporter.commit_distance.channel import (
+    Channel,
+    SaasTarget,
+    build_channels,
+)
+from tools.saas_metrics_exporter.commit_distance.metrics import SaasCommitDistanceGauge
+
+
+@dataclass
+class Distance:
+    publisher: SaasTarget
+    subscriber: SaasTarget
+    channel: Channel
+    distance: int = 0
+
+
+@dataclass
+class CommitDistanceMetric:
+    value: float
+    metric: SaasCommitDistanceGauge
+
+
+@dataclass
+class ThreadData:
+    repo_url: str
+    auth_code: HasSecret | None
+    ref_from: str
+    ref_to: str
+    distance: int = 0
+
+    def __hash__(self) -> int:
+        return hash((self.repo_url, self.ref_from, self.ref_to))
+
+
+class CommitDistanceFetcher:
+    def __init__(self, vcs: VCS):
+        self._vcs = vcs
+
+    def _data_key(self, repo_url: str, ref_from: str, ref_to: str) -> str:
+        return f"{repo_url}/{ref_from}/{ref_to}"
+
+    def _calculate_commit_distance(self, data: ThreadData) -> None:
+        if data.ref_from == data.ref_to:
+            data.distance = 0
+            return
+
+        commits = self._vcs.get_commits_between(
+            repo_url=data.repo_url,
+            auth_code=data.auth_code,
+            commit_from=data.ref_from,
+            commit_to=data.ref_to,
+        )
+        data.distance = len(commits)
+
+    def _populate_distances(
+        self, distances: Iterable[Distance], thread_data: Iterable[ThreadData]
+    ) -> None:
+        m = {
+            self._data_key(
+                repo_url=d.repo_url, ref_from=d.ref_from, ref_to=d.ref_to
+            ): d.distance
+            for d in thread_data
+        }
+        for distance in distances:
+            distance.distance = m[
+                self._data_key(
+                    repo_url=distance.publisher.repo_url,
+                    ref_from=distance.subscriber.ref,
+                    ref_to=distance.publisher.ref,
+                )
+            ]
+
+    def fetch(
+        self, saas_files: Iterable[SaasFile], thread_pool_size: int
+    ) -> list[CommitDistanceMetric]:
+        channels = build_channels(saas_files=saas_files)
+        distances: list[Distance] = []
+        thread_data: set[ThreadData] = set()
+
+        for channel in channels:
+            for subscriber in channel.subscribers:
+                for publisher in channel.publishers:
+                    thread_data.add(
+                        ThreadData(
+                            repo_url=publisher.repo_url,
+                            auth_code=publisher.auth_code,
+                            ref_from=subscriber.ref,
+                            ref_to=publisher.ref,
+                        )
+                    )
+                    distances.append(
+                        Distance(
+                            publisher=publisher,
+                            subscriber=subscriber,
+                            channel=channel,
+                        )
+                    )
+
+        threaded.run(
+            self._calculate_commit_distance,
+            thread_data,
+            thread_pool_size=thread_pool_size,
+        )
+
+        self._populate_distances(distances=distances, thread_data=thread_data)
+
+        commit_distance_metrics = [
+            CommitDistanceMetric(
+                value=float(distance.distance),
+                metric=SaasCommitDistanceGauge(
+                    channel=distance.channel.name,
+                    app=distance.publisher.app_name,
+                    publisher=distance.publisher.target_name,
+                    publisher_namespace=distance.publisher.namespace_name,
+                    subscriber=distance.subscriber.target_name,
+                    subscriber_namespace=distance.subscriber.namespace_name,
+                ),
+            )
+            for distance in distances
+        ]
+
+        return commit_distance_metrics

tools/saas_metrics_exporter/commit_distance/metrics.py

@@ -0,0 +1,26 @@
+from pydantic import BaseModel
+
+from reconcile.utils.metrics import (
+    GaugeMetric,
+)
+
+
+class SaasBaseMetric(BaseModel):
+    "Base class for Saas metrics"
+
+    integration: str = "saas_metrics_exporter"
+
+
+class SaasCommitDistanceGauge(SaasBaseMetric, GaugeMetric):
+    "Gauge for the commit distance between saas targets in a channel"
+
+    channel: str
+    publisher: str
+    publisher_namespace: str
+    subscriber: str
+    subscriber_namespace: str
+    app: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "commit_distance"

tools/saas_metrics_exporter/main.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+
+import click
+
+from reconcile.cli import (
+    config_file,
+    dry_run,
+    log_level,
+)
+from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.typed_queries.github_orgs import get_github_orgs
+from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
+from reconcile.typed_queries.saas_files import get_saas_files
+from reconcile.utils import metrics
+from reconcile.utils.defer import defer
+from reconcile.utils.runtime.environment import init_env
+from reconcile.utils.secret_reader import create_secret_reader
+from reconcile.utils.vcs import VCS
+from tools.saas_metrics_exporter.commit_distance.commit_distance import (
+    CommitDistanceFetcher,
+)
+
+
+class SaasMetricsExporter:
+    """
+    This tool is responsible for exposing/exporting saas metrics and data.
+
+    Note, that by design we store metrics exporters as a tool in the tools directory.
+    """
+
+    def __init__(self, vcs: VCS, dry_run: bool) -> None:
+        self._vcs = vcs
+        self._dry_run = dry_run
+
+    @staticmethod
+    def create(dry_run: bool) -> SaasMetricsExporter:
+        vault_settings = get_app_interface_vault_settings()
+        secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+        vcs = VCS(
+            secret_reader=secret_reader,
+            github_orgs=get_github_orgs(),
+            gitlab_instances=get_gitlab_instances(),
+            app_interface_repo_url=get_app_interface_repo_url(),
+            dry_run=dry_run,
+        )
+        return SaasMetricsExporter(vcs=vcs, dry_run=dry_run)
+
+    @defer
+    def run(
+        self,
+        env_name: str | None,
+        app_name: str | None,
+        thread_pool_size: int,
+        defer: Callable | None = None,
+    ) -> None:
+        saas_files = get_saas_files(env_name=env_name, app_name=app_name)
+        if defer:
+            defer(self._vcs.cleanup)
+
+        commit_distance_fetcher = CommitDistanceFetcher(vcs=self._vcs)
+        commit_distance_metrics = commit_distance_fetcher.fetch(
+            saas_files=saas_files, thread_pool_size=thread_pool_size
+        )
+        for m in commit_distance_metrics:
+            metrics.set_gauge(
+                metric=m.metric,
+                value=m.value,
+            )
+
+
+@click.command()
+@click.option("--env-name", default=None, help="environment to filter saas files by")
+@click.option("--app-name", default=None, help="app to filter saas files by")
+@click.option("--thread-pool-size", default=1, help="threadpool size")
+@dry_run
+@config_file
+@log_level
+def main(
+    env_name: str | None,
+    app_name: str | None,
+    dry_run: bool,
+    thread_pool_size: int,
+    configfile: str,
+    log_level: str | None,
+) -> None:
+    init_env(log_level=log_level, config_file=configfile)
+    exporter = SaasMetricsExporter.create(dry_run=dry_run)
+    exporter.run(
+        env_name=env_name, app_name=app_name, thread_pool_size=thread_pool_size
+    )
+
+
+if __name__ == "__main__":
+    main()  # pylint: disable=no-value-for-parameter

tools/test/conftest.py

@@ -0,0 +1,40 @@
+from collections.abc import (
+    Callable,
+    MutableMapping,
+)
+from typing import (
+    Any,
+    Optional,
+)
+
+import pytest
+from pydantic import BaseModel
+from pydantic.error_wrappers import ValidationError
+
+from reconcile.utils.models import data_default_none
+
+
+class GQLClassFactoryError(Exception):
+    pass
+
+
+@pytest.fixture
+def gql_class_factory() -> (
+    Callable[
+        [type[BaseModel], Optional[MutableMapping[str, Any]]],
+        BaseModel,
+    ]
+):
+    """Create a GQL class from a fixture and set default values to None."""
+
+    def _gql_class_factory(
+        klass: type[BaseModel], data: Optional[MutableMapping[str, Any]] = None
+    ) -> BaseModel:
+        try:
+            return klass(**data_default_none(klass, data or {}))
+        except ValidationError as e:
+            msg = "[gql_class_factory] Your given data does not match the class ...\n"
+            msg += "\n".join([str(m) for m in list(e.raw_errors)])
+            raise GQLClassFactoryError(msg) from e
+
+    return _gql_class_factory