qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

tools/saas_metrics_exporter/commit_distance/channel.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+from reconcile.typed_queries.saas_files import SaasFile
+from reconcile.utils.secret_reader import HasSecret
+
+
+@dataclass
+class SaasTarget:
+    app_name: str
+    repo_url: str
+    namespace_name: str
+    target_name: str
+    ref: str
+    auth_code: HasSecret | None
+
+
+@dataclass
+class Channel:
+    name: str
+    subscribers: list[SaasTarget]
+    publishers: list[SaasTarget]
+
+
+def build_channels(saas_files: Iterable[SaasFile]) -> list[Channel]:
+    channels: dict[str, Channel] = {}
+    for saas_file in saas_files:
+        for resource_template in saas_file.resource_templates:
+            for target in resource_template.targets:
+                if not target.promotion:
+                    continue
+                if not (target.promotion.publish or target.promotion.subscribe):
+                    continue
+                auth_code = (
+                    saas_file.authentication.code if saas_file.authentication else None
+                )
+                target_name = target.name if target.name else "NoName"
+                saas_target = SaasTarget(
+                    app_name=saas_file.app.name,
+                    repo_url=resource_template.url,
+                    ref=target.ref,
+                    auth_code=auth_code,
+                    namespace_name=target.namespace.name,
+                    target_name=target_name,
+                )
+
+                for channel in target.promotion.publish or []:
+                    if channel not in channels:
+                        channels[channel] = Channel(
+                            name=channel, subscribers=[], publishers=[]
+                        )
+                    channels[channel].publishers.append(saas_target)
+
+                for channel in target.promotion.subscribe or []:
+                    if channel not in channels:
+                        channels[channel] = Channel(
+                            name=channel, subscribers=[], publishers=[]
+                        )
+                    channels[channel].subscribers.append(saas_target)
+
+    return list(channels.values())

tools/saas_metrics_exporter/commit_distance/commit_distance.py

@@ -0,0 +1,103 @@
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+
+from sretoolbox.utils import threaded
+
+from reconcile.typed_queries.saas_files import SaasFile
+from reconcile.utils.secret_reader import HasSecret
+from reconcile.utils.vcs import VCS
+from tools.saas_metrics_exporter.commit_distance.channel import (
+    SaasTarget,
+    build_channels,
+)
+from tools.saas_metrics_exporter.commit_distance.metrics import SaasCommitDistanceGauge
+
+
+@dataclass
+class CommitDistanceMetric:
+    value: float
+    metric: SaasCommitDistanceGauge
+
+
+@dataclass(frozen=True)
+class DistanceKey:
+    repo_url: str
+    auth_code: HasSecret | None = field(hash=False, compare=False)
+    ref_from: str
+    ref_to: str
+
+
+class CommitDistanceFetcher:
+    def __init__(self, vcs: VCS):
+        self._vcs = vcs
+
+    def _calculate_commit_distance(self, key: DistanceKey) -> tuple[DistanceKey, int]:
+        if key.ref_from == key.ref_to:
+            return key, 0
+
+        commits = self._vcs.get_commits_between(
+            repo_url=key.repo_url,
+            auth_code=key.auth_code,
+            commit_from=key.ref_from,
+            commit_to=key.ref_to,
+        )
+        return key, len(commits)
+
+    @staticmethod
+    def _build_distance_key(
+        publisher: SaasTarget, subscriber: SaasTarget
+    ) -> DistanceKey:
+        return DistanceKey(
+            repo_url=publisher.repo_url,
+            auth_code=publisher.auth_code,
+            ref_from=subscriber.ref,
+            ref_to=publisher.ref,
+        )
+
+    def fetch(
+        self,
+        saas_files: Iterable[SaasFile],
+        thread_pool_size: int,
+    ) -> list[CommitDistanceMetric]:
+        channels = build_channels(saas_files=saas_files)
+
+        distance_keys = {
+            self._build_distance_key(publisher=publisher, subscriber=subscriber)
+            for channel in channels
+            for subscriber in channel.subscribers
+            for publisher in channel.publishers
+        }
+
+        distance_by_key = dict(
+            threaded.run(
+                self._calculate_commit_distance,
+                distance_keys,
+                thread_pool_size=thread_pool_size,
+            )
+        )
+
+        commit_distance_metrics = [
+            CommitDistanceMetric(
+                value=float(
+                    distance_by_key[
+                        self._build_distance_key(
+                            publisher=publisher,
+                            subscriber=subscriber,
+                        )
+                    ]
+                ),
+                metric=SaasCommitDistanceGauge(
+                    channel=channel.name,
+                    app=publisher.app_name,
+                    publisher=publisher.target_name,
+                    publisher_namespace=publisher.namespace_name,
+                    subscriber=subscriber.target_name,
+                    subscriber_namespace=subscriber.namespace_name,
+                ),
+            )
+            for channel in channels
+            for subscriber in channel.subscribers
+            for publisher in channel.publishers
+        ]
+
+        return commit_distance_metrics

tools/saas_metrics_exporter/commit_distance/metrics.py

@@ -0,0 +1,19 @@
+from reconcile.utils.metrics import (
+    GaugeMetric,
+)
+
+
+class SaasCommitDistanceGauge(GaugeMetric):
+    "Gauge for the commit distance between saas targets in a channel"
+
+    integration: str = "saas_metrics_exporter"
+    channel: str
+    publisher: str
+    publisher_namespace: str
+    subscriber: str
+    subscriber_namespace: str
+    app: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "commit_distance"

tools/saas_metrics_exporter/main.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+
+import click
+
+from reconcile.cli import (
+    config_file,
+    dry_run,
+    log_level,
+)
+from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.typed_queries.github_orgs import get_github_orgs
+from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
+from reconcile.typed_queries.saas_files import get_saas_files
+from reconcile.utils import metrics
+from reconcile.utils.defer import defer
+from reconcile.utils.runtime.environment import init_env
+from reconcile.utils.secret_reader import create_secret_reader
+from reconcile.utils.vcs import VCS
+from tools.saas_metrics_exporter.commit_distance.commit_distance import (
+    CommitDistanceFetcher,
+)
+
+
+class SaasMetricsExporter:
+    """
+    This tool is responsible for exposing/exporting saas metrics and data.
+
+    Note, that by design we store metrics exporters as a tool in the tools directory.
+    """
+
+    def __init__(self, vcs: VCS, dry_run: bool) -> None:
+        self._vcs = vcs
+        self._dry_run = dry_run
+
+    @staticmethod
+    def create(dry_run: bool) -> SaasMetricsExporter:
+        vault_settings = get_app_interface_vault_settings()
+        secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+        vcs = VCS(
+            secret_reader=secret_reader,
+            github_orgs=get_github_orgs(),
+            gitlab_instances=get_gitlab_instances(),
+            app_interface_repo_url=get_app_interface_repo_url(),
+            dry_run=dry_run,
+        )
+        return SaasMetricsExporter(vcs=vcs, dry_run=dry_run)
+
+    @defer
+    def run(
+        self,
+        env_name: str | None,
+        app_name: str | None,
+        thread_pool_size: int,
+        defer: Callable | None = None,
+    ) -> None:
+        saas_files = get_saas_files(env_name=env_name, app_name=app_name)
+        if defer:
+            defer(self._vcs.cleanup)
+
+        commit_distance_fetcher = CommitDistanceFetcher(vcs=self._vcs)
+        commit_distance_metrics = commit_distance_fetcher.fetch(
+            saas_files=saas_files, thread_pool_size=thread_pool_size
+        )
+        for m in commit_distance_metrics:
+            metrics.set_gauge(
+                metric=m.metric,
+                value=m.value,
+            )
+
+
+@click.command()
+@click.option("--env-name", default=None, help="environment to filter saas files by")
+@click.option("--app-name", default=None, help="app to filter saas files by")
+@click.option("--thread-pool-size", default=1, help="threadpool size")
+@dry_run
+@config_file
+@log_level
+def main(
+    env_name: str | None,
+    app_name: str | None,
+    dry_run: bool,
+    thread_pool_size: int,
+    configfile: str,
+    log_level: str | None,
+) -> None:
+    init_env(log_level=log_level, config_file=configfile)
+    exporter = SaasMetricsExporter.create(dry_run=dry_run)
+    exporter.run(
+        env_name=env_name, app_name=app_name, thread_pool_size=thread_pool_size
+    )
+
+
+if __name__ == "__main__":
+    main()  # pylint: disable=no-value-for-parameter

tools/saas_promotion_state/saas_promotion_state.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+from reconcile.openshift_saas_deploy import (
+    QONTRACT_INTEGRATION as OPENSHIFT_SAAS_DEPLOY,
+)
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.typed_queries.saas_files import SaasFile, get_saas_files
+from reconcile.utils.promotion_state import PromotionData, PromotionState
+from reconcile.utils.secret_reader import create_secret_reader
+from reconcile.utils.state import init_state
+
+
+class SaasPromotionStateException(Exception):
+    pass
+
+
+class SaasPromotionStateMissingException(Exception):
+    pass
+
+
+class SaasPromotionState:
+    def __init__(
+        self, promotion_state: PromotionState, saas_files: Iterable[SaasFile]
+    ) -> None:
+        self._promotion_state = promotion_state
+        self._saas_files = saas_files
+
+    def _publisher_ids_for_channel(
+        self, channel: str, saas_files: Iterable[SaasFile]
+    ) -> list[str]:
+        publisher_uids: list[str] = []
+        for saas_file in saas_files:
+            for resource_template in saas_file.resource_templates:
+                for target in resource_template.targets:
+                    if not target.promotion:
+                        continue
+                    for publish_channel in target.promotion.publish or []:
+                        if publish_channel == channel:
+                            publisher_uids.append(
+                                target.uid(
+                                    parent_saas_file_name=saas_file.name,
+                                    parent_resource_template_name=resource_template.name,
+                                )
+                            )
+        return publisher_uids
+
+    def get(self, channel: str, sha: str) -> dict[str, PromotionData | None]:
+        return {
+            publisher_id: self._promotion_state.get_promotion_data(
+                sha=sha,
+                channel=channel,
+                use_cache=False,
+                target_uid=publisher_id,
+                pre_check_sha_exists=False,
+            )
+            for publisher_id in self._publisher_ids_for_channel(
+                channel=channel, saas_files=self._saas_files
+            )
+        }
+
+    def set_successful(self, channel: str, sha: str, publisher_uid: str) -> None:
+        current_data = self._promotion_state.get_promotion_data(
+            sha=sha,
+            channel=channel,
+            target_uid=publisher_uid,
+            use_cache=False,
+            pre_check_sha_exists=False,
+        )
+
+        if not current_data:
+            raise SaasPromotionStateMissingException(
+                f"No promotion state in S3 for given {publisher_uid=} {sha=} {channel=}"
+            )
+
+        if current_data.success:
+            raise SaasPromotionStateException(
+                f"The current promotion state is already marked successful for given {publisher_uid=} {sha=} {channel=}",
+                current_data,
+            )
+
+        current_data.success = True
+        self._promotion_state.publish_promotion_data(
+            data=current_data, sha=sha, channel=channel, target_uid=publisher_uid
+        )
+
+    @staticmethod
+    def create(
+        promotion_state: PromotionState | None, saas_files: Iterable[SaasFile] | None
+    ) -> SaasPromotionState:
+        if not promotion_state:
+            vault_settings = get_app_interface_vault_settings()
+            secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+            saas_deploy_state = init_state(
+                integration=OPENSHIFT_SAAS_DEPLOY, secret_reader=secret_reader
+            )
+            promotion_state = PromotionState(state=saas_deploy_state)
+        if not saas_files:
+            saas_files = get_saas_files()
+        return SaasPromotionState(
+            promotion_state=promotion_state, saas_files=saas_files
+        )

tools/sd_app_sre_alert_report.py

@@ -0,0 +1,145 @@
+import logging
+import re
+from collections import defaultdict
+from dataclasses import dataclass
+
+QONTRACT_INTEGRATION = "sd-app-sre-alert-report"
+
+
+@dataclass
+class Alert:
+    state: str
+    message: str
+    timestamp: float
+    username: str
+
+
+class AlertStat:
+    def __init__(self) -> None:
+        self._triggered_alerts = 0
+        self._resolved_alerts = 0
+        self._elapsed_times: list[float] = []
+
+    def add_triggered(self) -> None:
+        self._triggered_alerts += 1
+
+    def add_resolved(self) -> None:
+        self._resolved_alerts += 1
+
+    def add_elapsed_time(self, elapsed: float) -> None:
+        self._elapsed_times.append(elapsed)
+
+    @property
+    def triggered_alerts(self) -> int:
+        return self._triggered_alerts
+
+    @property
+    def resolved_alerts(self) -> int:
+        return self._resolved_alerts
+
+    @property
+    def elapsed_times(self) -> list[float]:
+        return self._elapsed_times
+
+
+def group_alerts(messages: list[dict]) -> dict[str, list[Alert]]:
+    """Group list of alert messages from Slack in a dict indexed by alert_name"""
+    alerts = defaultdict(list)
+    for m in messages:
+        if "subtype" not in m or m["subtype"] != "bot_message":
+            logging.debug(
+                f'Skipping message \'{m["text"]}\' as it does not come from a bot'
+            )
+            continue
+
+        timestamp = float(m["ts"])
+        for at in m.get("attachments", []):
+            if "title" not in at:
+                continue
+
+            mg = re.match(r"Alert: (.*) \[(FIRING:\d+|RESOLVED)\] *(.*)$", at["title"])
+            if not mg:
+                continue
+
+            alert_name = mg.group(1)
+            alert_message = mg.group(3)
+
+            if not alert_name:
+                logging.debug(f'no alert name in title {at["title"]}. Skipping')
+                continue
+
+            # If there's only one alert related to the alert_name, message will be part
+            # of the title. If not, alert messages will be part of the text under
+            # "Alerts Firing" / "Alerts Resolved"
+            if alert_message:
+                alert_state = "FIRING" if "FIRING" in mg.group(2) else mg.group(2)
+                alerts[alert_name].append(
+                    Alert(
+                        state=alert_state,
+                        message=alert_message,
+                        timestamp=timestamp,
+                        username=m["username"],
+                    )
+                )
+            else:
+                # This may happen for alerts rules without "message". This can happen
+                # if schema cannot be validated for a certain alert rule because it
+                # is not valid yaml (go templates, jinja templates)
+                if "text" not in at:
+                    alert_state = "FIRING" if "FIRING" in mg.group(2) else mg.group(2)
+                    alerts[alert_name].append(
+                        Alert(
+                            state=alert_state,
+                            message="placeholder",
+                            timestamp=timestamp,
+                            username=m["username"],
+                        )
+                    )
+                    continue
+
+                alert_state = ""
+                for line in at["text"].split("\n"):
+                    if "Alerts Firing" in line:
+                        alert_state = "FIRING"
+                    elif "Alerts Resolved" in line:
+                        alert_state = "RESOLVED"
+                    elif line.startswith("-"):
+                        mg = re.match("^- (.+)$", line)
+                        if not mg:
+                            continue
+                        alert_message = mg.group(1)
+                        alerts[alert_name].append(
+                            Alert(
+                                state=alert_state,
+                                message=alert_message,
+                                timestamp=timestamp,
+                                username=m["username"],
+                            )
+                        )
+
+    return dict(alerts)
+
+
+def gen_alert_stats(alerts: dict[str, list[Alert]]) -> dict[str, AlertStat]:
+    """Get the parsed alerts dict and measure elapsed times until resolved by grouping
+    them by alert message and cluster where the message comes from"""
+    alert_stats = {}
+    for alert_name, alert_list in alerts.items():
+        temp = {}
+        alert_stats[alert_name] = AlertStat()
+
+        # Slack api returns messages in reversed order, newests first.
+        for al in reversed(alert_list):
+            key = (al.message, al.username)  # username is different per cluster
+
+            if al.state == "FIRING":
+                alert_stats[alert_name].add_triggered()
+                temp[key] = al
+
+            if al.state == "RESOLVED":
+                alert_stats[alert_name].add_resolved()
+                if key in temp:
+                    elapsed = al.timestamp - temp[key].timestamp
+                    alert_stats[alert_name].add_elapsed_time(elapsed)
+
+    return alert_stats

tools/template_validation.py

@@ -0,0 +1,107 @@
+import json
+import os.path
+import sys
+
+import click
+from yamllint import linter  # type: ignore
+from yamllint.config import YamlLintConfig  # type: ignore
+
+from reconcile.gql_definitions.templating.templates import TemplateV1
+from reconcile.templating.validator import TemplateDiff, TemplateValidatorIntegration
+from reconcile.utils.models import data_default_none
+from reconcile.utils.ruamel import create_ruamel_instance
+
+
+def load_clean_yaml(path: str) -> dict:
+    if not path.startswith("data/"):
+        to_load = os.path.join("data", path.lstrip("/"))
+    else:
+        to_load = os.path.join(path.lstrip("/"))
+
+    y = load_yaml(to_load)
+
+    if "$schema" in y:
+        del y["$schema"]
+    return y
+
+
+def load_yaml(to_load: str) -> dict:
+    ruamel_instance = create_ruamel_instance()
+    with open(to_load, encoding="utf-8") as file:
+        return ruamel_instance.load(file)
+
+
+def print_lint_problems(lint_problems: list[linter.LintProblem]) -> None:
+    print("--------- LINTING ISSUES ---------")
+    print(f"Found {len(lint_problems)} lint problems:")
+    for problem in lint_problems:
+        print(f"Lint error in line: {problem.line}, {problem.desc}")
+    print("--------- END LINTING ISSUES ---------")
+
+
+def print_test_diffs(diffs: list[TemplateDiff]) -> None:
+    print("--------- TEMPLATE TEST DIFF ---------")
+    print("Template validation failed")
+    for d in diffs:
+        for line in d.diff.splitlines():
+            if line:
+                print(f"{line}")
+    print("--------- END TEMPLATE TEST DIFF ---------")
+
+
+@click.command()
+@click.argument(
+    "templates",
+    nargs=-1,
+)
+def main(templates: tuple[str]) -> None:
+    for template_path in templates:
+        okay = True
+        templateRaw = load_clean_yaml(template_path)
+
+        tests = []
+        for testRaw in templateRaw["templateTest"]:
+            test_yaml = load_clean_yaml(testRaw["$ref"])
+            variables = json.dumps(test_yaml["variables"])
+            test_yaml["variables"] = variables
+            tests.append(test_yaml)
+
+        templateRaw["templateTest"] = tests
+        template: TemplateV1 = TemplateV1(**data_default_none(TemplateV1, templateRaw))
+
+        # templates_to_validate = {}
+        for test in template.template_test:
+            ruaml_instance = create_ruamel_instance(explicit_start=True)
+            diffs: list[TemplateDiff] = []
+            print("Running tests:", test.name)
+            diffs.extend(
+                TemplateValidatorIntegration.validate_template(
+                    template, test, ruaml_instance=ruaml_instance
+                )
+            )
+
+            renderer = TemplateValidatorIntegration._create_renderer(
+                template, test, ruaml_instance=ruaml_instance
+            )
+            if renderer.render_condition():
+                output = renderer.render_output()
+                lint_problems = list(
+                    linter.run(output, YamlLintConfig(file=".yamllint"), "")
+                )
+                if lint_problems:
+                    okay = False
+                    print_lint_problems(lint_problems)
+
+            if diffs:
+                okay = False
+                print_test_diffs(diffs)
+
+    if okay:
+        print("... passed")
+        sys.exit(0)
+    print("... failed")
+    sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()  # pylint: disable=no-value-for-parameter