qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/saas_auto_promotions_manager/merge_request_manager/mr_parser.py

@@ -0,0 +1,226 @@
+import logging
+import re
+from collections.abc import Iterable
+
+from gitlab.v4.objects import ProjectMergeRequest
+
+from reconcile.saas_auto_promotions_manager.merge_request_manager.open_merge_requests import (
+    MRKind,
+    OpenBatcherMergeRequest,
+    OpenSchedulerMergeRequest,
+)
+from reconcile.saas_auto_promotions_manager.merge_request_manager.renderer import (
+    CHANNELS_REF,
+    CONTENT_HASHES,
+    IS_BATCHABLE,
+    MR_KIND_REF,
+    PROMOTION_DATA_SEPARATOR,
+    SAPM_VERSION,
+    VERSION_REF,
+)
+from reconcile.utils.vcs import VCS, MRCheckStatus
+
+ITEM_SEPARATOR = ","
+
+
+class MRParser:
+    """
+    We store state in MR descriptions.
+
+    This class parses open MRs and ensures they are compatible with our logic.
+    I.e., this class fetches the current state.
+    """
+
+    def __init__(self, vcs: VCS):
+        self._vcs = vcs
+        self._version_ref_regex = re.compile(rf"{VERSION_REF}: (.*)$", re.MULTILINE)
+        self._content_hash_regex = re.compile(rf"{CONTENT_HASHES}: (.*)$", re.MULTILINE)
+        self._channels_regex = re.compile(rf"{CHANNELS_REF}: (.*)$", re.MULTILINE)
+        self._is_batchable_regex = re.compile(rf"{IS_BATCHABLE}: (.*)$", re.MULTILINE)
+        self._mr_kind_regex = re.compile(rf"{MR_KIND_REF}: (.*)$", re.MULTILINE)
+        self._open_batcher_mrs: list[OpenBatcherMergeRequest] = []
+        self._open_scheduler_mrs: list[OpenSchedulerMergeRequest] = []
+
+    def get_open_batcher_mrs(self) -> list[OpenBatcherMergeRequest]:
+        return self._open_batcher_mrs
+
+    def get_open_scheduler_mrs(self) -> list[OpenSchedulerMergeRequest]:
+        return self._open_scheduler_mrs
+
+    def _apply_regex(self, pattern: re.Pattern, promotion_data: str) -> str:
+        matches = pattern.search(promotion_data)
+        if not matches:
+            return ""
+        groups = matches.groups()
+        if len(groups) != 1:
+            return ""
+        return groups[0]
+
+    def fetch_mrs(self, label: str) -> None:
+        """
+        This function parses the state of valid, parsed open MRs (current state).
+        If any issue is encountered during parsing, we consider this MR
+        to be broken and close it. Information we want to parse includes:
+        - SAPM_VERSION -> Close if it doesnt match current version
+        - CHANNELS
+        - CONTENT_HASHES
+        - IS_BATCHABLE flag
+        - MR has merge conflicts
+        """
+        all_open_mrs = self._vcs.get_open_app_interface_merge_requests()
+        sapm_mrs = [
+            mr for mr in all_open_mrs if label in mr.attributes.get("labels", [])
+        ]
+        open_batcher_mrs: list[ProjectMergeRequest] = []
+        open_scheduler_mrs: list[ProjectMergeRequest] = []
+
+        for mr in sapm_mrs:
+            attrs = mr.attributes
+            desc = str(attrs.get("description", ""))
+            parts = desc.split(PROMOTION_DATA_SEPARATOR)
+            if len(parts) != 2:
+                logging.info(
+                    "Bad data separator format. Closing %s",
+                    mr.attributes.get("web_url", "NO_WEBURL"),
+                )
+                self._vcs.close_app_interface_mr(
+                    mr, "Closing this MR because of bad data separator format."
+                )
+                continue
+            promotion_data = parts[1]
+
+            version_ref = self._apply_regex(
+                pattern=self._version_ref_regex, promotion_data=promotion_data
+            )
+            if not version_ref:
+                logging.info(
+                    "Bad %s format. Closing %s",
+                    VERSION_REF,
+                    mr.attributes.get("web_url", "NO_WEBURL"),
+                )
+                self._vcs.close_app_interface_mr(
+                    mr, f"Closing this MR because of bad {VERSION_REF} format."
+                )
+                continue
+
+            if version_ref != SAPM_VERSION:
+                logging.info(
+                    "Old MR version detected: %s. Closing %s",
+                    version_ref,
+                    mr.attributes.get("web_url", "NO_WEBURL"),
+                )
+                self._vcs.close_app_interface_mr(
+                    mr,
+                    f"Closing this MR because it has an outdated SAPM version {version_ref}.",
+                )
+                continue
+
+            mr_kind_str = self._apply_regex(
+                pattern=self._mr_kind_regex, promotion_data=promotion_data
+            )
+            if not mr_kind_str or mr_kind_str not in {
+                MRKind.BATCHER.value,
+                MRKind.SCHEDULER.value,
+            }:
+                logging.info(
+                    "Bad %s format. Closing %s",
+                    MR_KIND_REF,
+                    mr.attributes.get("web_url", "NO_WEBURL"),
+                )
+                self._vcs.close_app_interface_mr(
+                    mr, f"Closing this MR because of bad {MR_KIND_REF} format."
+                )
+                continue
+
+            mr_kind = MRKind(mr_kind_str)
+            if mr_kind == MRKind.BATCHER:
+                open_batcher_mrs.append(mr)
+            elif mr_kind == MRKind.SCHEDULER:
+                open_scheduler_mrs.append(mr)
+
+        self._handle_open_batcher_mrs(open_batcher_mrs)
+
+    def _handle_open_batcher_mrs(self, mrs: Iterable[ProjectMergeRequest]) -> None:
+        seen: set[tuple[str, str]] = set()
+        for mr in mrs:
+            attrs = mr.attributes
+            desc = str(attrs.get("description", ""))
+            parts = desc.split(PROMOTION_DATA_SEPARATOR)
+            promotion_data = parts[1]
+            has_conflicts = bool(attrs.get("has_conflicts", False))
+            if has_conflicts:
+                logging.info(
+                    "Merge-conflict detected. Closing %s",
+                    mr.attributes.get("web_url", "NO_WEBURL"),
+                )
+                self._vcs.close_app_interface_mr(
+                    mr, "Closing this MR because of a merge-conflict."
+                )
+                continue
+
+            content_hashes = self._apply_regex(
+                pattern=self._content_hash_regex, promotion_data=promotion_data
+            )
+            if not content_hashes:
+                logging.info(
+                    "Bad %s format. Closing %s",
+                    CONTENT_HASHES,
+                    mr.attributes.get("web_url", "NO_WEBURL"),
+                )
+                self._vcs.close_app_interface_mr(
+                    mr, f"Closing this MR because of bad {CONTENT_HASHES} format."
+                )
+                continue
+
+            channels_refs = self._apply_regex(
+                pattern=self._channels_regex, promotion_data=promotion_data
+            )
+            if not channels_refs:
+                logging.info(
+                    "Bad %s format. Closing %s",
+                    CHANNELS_REF,
+                    mr.attributes.get("web_url", "NO_WEBURL"),
+                )
+                self._vcs.close_app_interface_mr(
+                    mr, f"Closing this MR because of bad {CHANNELS_REF} format."
+                )
+                continue
+
+            key = (channels_refs, content_hashes)
+            if key in seen:
+                logging.info(
+                    "Duplicate MR detected. Closing %s",
+                    mr.attributes.get("web_url", "NO_WEBURL"),
+                )
+                self._vcs.close_app_interface_mr(
+                    mr,
+                    "Closing this MR because there is already another MR open with identical content.",
+                )
+                continue
+            seen.add(key)
+
+            is_batchable_str = self._apply_regex(
+                pattern=self._is_batchable_regex, promotion_data=promotion_data
+            )
+            if is_batchable_str not in {"True", "False"}:
+                logging.info(
+                    "Bad %s format. Closing %s",
+                    IS_BATCHABLE,
+                    mr.attributes.get("web_url", "NO_WEBURL"),
+                )
+                self._vcs.close_app_interface_mr(
+                    mr, f"Closing this MR because of bad {IS_BATCHABLE} format."
+                )
+                continue
+
+            mr_check_status = self._vcs.get_gitlab_mr_check_status(mr)
+
+            self._open_batcher_mrs.append(
+                OpenBatcherMergeRequest(
+                    raw=mr,
+                    content_hashes=set(content_hashes.split(ITEM_SEPARATOR)),
+                    channels=set(channels_refs.split(ITEM_SEPARATOR)),
+                    failed_mr_check=mr_check_status == MRCheckStatus.FAILED,
+                    is_batchable=is_batchable_str == "True",
+                )
+            )

reconcile/saas_auto_promotions_manager/merge_request_manager/open_merge_requests.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass
+from enum import Enum
+
+from gitlab.v4.objects import ProjectMergeRequest
+
+
+class MRKind(Enum):
+    BATCHER = "batcher"
+    SCHEDULER = "scheduler"
+
+
+@dataclass
+class OpenBatcherMergeRequest:
+    raw: ProjectMergeRequest
+    content_hashes: set[str]
+    channels: set[str]
+    failed_mr_check: bool
+    is_batchable: bool
+
+
+@dataclass
+class OpenSchedulerMergeRequest:
+    pass

reconcile/saas_auto_promotions_manager/merge_request_manager/renderer.py

@@ -1,25 +1,34 @@
-from ruamel.yaml import YAML
+import hashlib
+from collections.abc import Mapping
+from copy import deepcopy
+from typing import Any
+
+from jsonpath_ng.exceptions import JsonPathParserError
+from jsonpath_ng.ext import parser
 from ruamel.yaml.compat import StringIO
 
+from reconcile.gql_definitions.common.saas_files import (
+    SaasResourceTemplateTargetNamespaceSelectorV1,
+)
+from reconcile.gql_definitions.fragments.saas_target_namespace import (
+    SaasTargetNamespace,
+)
+from reconcile.saas_auto_promotions_manager.merge_request_manager.open_merge_requests import (
+    MRKind,
+)
+from reconcile.saas_auto_promotions_manager.meta import QONTRACT_INTEGRATION_VERSION
 from reconcile.saas_auto_promotions_manager.subscriber import Subscriber
+from reconcile.utils.ruamel import create_ruamel_instance
 
 PROMOTION_DATA_SEPARATOR = (
     "**SAPM Data - DO NOT MANUALLY CHANGE ANYTHING BELOW THIS LINE**"
 )
-SAPM_VERSION = "1.0.0"
-SAPM_LABEL = "SAPM"
-CONTENT_HASH = "content_hash"
+SAPM_VERSION = QONTRACT_INTEGRATION_VERSION
+CONTENT_HASHES = "content_hashes"
 CHANNELS_REF = "channels"
+IS_BATCHABLE = "is_batchable"
 VERSION_REF = "sapm_version"
-SAPM_DESC = f"""
-This is an auto-promotion triggered by app-interface's [saas-auto-promotions-manager](https://github.com/app-sre/qontract-reconcile/tree/master/reconcile/saas_auto_promotions_manager) (SAPM).
-The channel(s) mentioned in the MR title had an event.
-This MR promotes all subscribers with auto-promotions for these channel(s).
-
-Please **do not remove the {SAPM_LABEL} label** from this MR.
-
-Parts of this description are used by SAPM to manage auto-promotions.
-"""
+MR_KIND_REF = "kind"
 
 
 class Renderer:
@@ -27,8 +36,29 @@ class Renderer:
     This class is only concerned with rendering text for MRs.
     Most logic evolves around ruamel yaml modification.
     This class makes testing for MergeRequestManager easier.
+
+    Note, that this class is very susceptible to schema changes
+    as it mainly works on raw dicts.
     """
 
+    def _is_wanted_target(
+        self, subscriber: Subscriber, target: Mapping[str, Any]
+    ) -> bool:
+        namespace_selector = deepcopy(target.get("namespaceSelector"))
+        if namespace_selector:
+            # We want to fail hard here on missing key - this should not happen
+            if "exclude" not in namespace_selector["jsonPathSelectors"]:
+                namespace_selector["jsonPathSelectors"]["exclude"] = []
+            selector = SaasResourceTemplateTargetNamespaceSelectorV1(
+                **namespace_selector
+            )
+            # Check if the target namespace is addressed by the selector
+            return is_namespace_addressed_by_selector(
+                namespace=subscriber.target_namespace,
+                namespace_selector=selector,
+            )
+        return target["namespace"]["$ref"] == subscriber.target_namespace.path
+
     def _find_saas_file_targets(
         self, subscriber: Subscriber, content: dict
     ) -> list[dict]:
@@ -38,7 +68,7 @@ class Renderer:
             return [content]
         for rt in content["resourceTemplates"]:
             for target in rt["targets"]:
-                if target["namespace"]["$ref"] != subscriber.namespace_file_path:
+                if not self._is_wanted_target(subscriber=subscriber, target=target):
                     continue
                 target_promotion = target.get("promotion")
                 if not target_promotion:
@@ -61,7 +91,7 @@ class Renderer:
         """
         # TODO: make prettier
         # this function is hell - but well tested
-        yml = YAML(typ="rt", pure=True)
+        yml = create_ruamel_instance(pure=True)
         content = yml.load(current_content)
         targets = self._find_saas_file_targets(subscriber=subscriber, content=content)
         for target in targets:
@@ -80,18 +110,16 @@ class Renderer:
                         applied_desired_hash = True
                 if not applied_desired_hash:
                     # This data block is not part of the promotion data yet -> add it
-                    cur_promotion_data.append(
-                        {
-                            "channel": desired_config_hash.channel,
-                            "data": [
-                                {
-                                    "parent_saas": desired_config_hash.parent_saas,
-                                    "target_config_hash": desired_config_hash.target_config_hash,
-                                    "type": "parent_saas_config",
-                                }
-                            ],
-                        }
-                    )
+                    cur_promotion_data.append({
+                        "channel": desired_config_hash.channel,
+                        "data": [
+                            {
+                                "parent_saas": desired_config_hash.parent_saas,
+                                "target_config_hash": desired_config_hash.target_config_hash,
+                                "type": "parent_saas_config",
+                            }
+                        ],
+                    })
             if cur_promotion_data:
                 target["promotion"]["promotion_data"] = cur_promotion_data
         new_content = "---\n"
@@ -100,18 +128,66 @@ class Renderer:
             new_content += stream.getvalue() or ""
         return new_content
 
-    def render_description(self, content_hash: str, channels: str) -> str:
+    def render_description(
+        self, message: str, content_hashes: str, channels: str, is_batchable: bool
+    ) -> str:
         return f"""
-{SAPM_DESC}
+{message}
 
 {PROMOTION_DATA_SEPARATOR}
 
 {CHANNELS_REF}: {channels}
 
-{CONTENT_HASH}: {content_hash}
+{CONTENT_HASHES}: {content_hashes}
+
+{IS_BATCHABLE}: {is_batchable}
+
+{MR_KIND_REF}: {MRKind.BATCHER.value}
 
 {VERSION_REF}: {SAPM_VERSION}
+
+This MR is auto-approved
+
+/lgtm
         """
 
-    def render_title(self, channels: str) -> str:
-        return f"[auto-promotion] event for channel(s) {channels}"
+    def render_title(
+        self, is_draft: bool, canary: bool, channels: str, batch_size: int
+    ) -> str:
+        canary_suffix = "Canary" if canary else ""
+        content = f"[SAPM{canary_suffix}] auto-promotion ID {int(hashlib.sha256(channels.encode('utf-8')).hexdigest(), 16) % 10**8} - batch size: {batch_size}"
+        if is_draft:
+            return f"Draft: {content}"
+        return content
+
+
+def _parse_expression(expression: str) -> Any:
+    try:
+        return parser.parse(expression)
+    except JsonPathParserError as e:
+        raise RuntimeError(
+            f"Invalid jsonpath expression in namespaceSelector '{expression}' :{e}"
+        ) from None
+
+
+def is_namespace_addressed_by_selector(
+    namespace: SaasTargetNamespace,
+    namespace_selector: SaasResourceTemplateTargetNamespaceSelectorV1,
+) -> bool:
+    # json representation of namespace to filter on
+    # remove all the None values to simplify the jsonpath expressions
+    namespace_as_dict = {
+        "namespace": [namespace.dict(by_alias=True, exclude_none=True)]
+    }
+
+    do_include = any(
+        _parse_expression(include).find(namespace_as_dict)
+        for include in namespace_selector.json_path_selectors.include or []
+    )
+
+    do_exclude = any(
+        _parse_expression(exclude).find(namespace_as_dict)
+        for exclude in namespace_selector.json_path_selectors.exclude or []
+    )
+
+    return do_include and not do_exclude

reconcile/saas_auto_promotions_manager/meta.py

@@ -0,0 +1,4 @@
+from reconcile.utils.semver_helper import make_semver
+
+QONTRACT_INTEGRATION = "saas-auto-promotions-manager"
+QONTRACT_INTEGRATION_VERSION = make_semver(2, 3, 0)

reconcile/saas_auto_promotions_manager/publisher.py

@@ -1,9 +1,9 @@
 from dataclasses import dataclass
-from typing import Optional
+from datetime import datetime
 
-from reconcile.saas_auto_promotions_manager.utils.vcs import VCS
 from reconcile.utils.promotion_state import PromotionState
 from reconcile.utils.secret_reader import HasSecret
+from reconcile.utils.vcs import VCS
 
 
 @dataclass
@@ -17,6 +17,7 @@ class DeploymentInfo:
     success: bool
     saas_file: str
     target_config_hash: str
+    check_in: datetime | None
 
 
 class Publisher:
@@ -29,14 +30,34 @@ class Publisher:
         self,
         ref: str,
         repo_url: str,
-        auth_code: Optional[HasSecret],
+        uid: str,
+        saas_name: str,
+        saas_file_path: str,
+        app_name: str,
+        namespace_name: str,
+        resource_template_name: str,
+        target_name: str | None,
+        cluster_name: str,
+        auth_code: HasSecret | None,
+        redeploy_on_config_change: bool | None,
+        has_subscriber: bool = True,
     ):
         self._ref = ref
         self._repo_url = repo_url
         self._auth_code = auth_code
         self.channels: set[str] = set()
         self.commit_sha: str = ""
-        self.deployment_info_by_channel: dict[str, Optional[DeploymentInfo]] = {}
+        self.deployment_info_by_channel: dict[str, DeploymentInfo | None] = {}
+        self.uid = uid
+        self.saas_name = saas_name
+        self.saas_file_path = saas_file_path
+        self.app_name = app_name
+        self.namespace_name = namespace_name
+        self.resource_template_name = resource_template_name
+        self.target_name = target_name if target_name else "None"
+        self.cluster_name = cluster_name
+        self.redeploy_on_config_change = bool(redeploy_on_config_change)
+        self.has_subscriber = has_subscriber
 
     def fetch_commit_shas_and_deployment_info(
         self, vcs: VCS, deployment_state: PromotionState
@@ -51,6 +72,7 @@ class Publisher:
             promotion_data = deployment_state.get_promotion_data(
                 sha=self.commit_sha,
                 channel=channel,
+                target_uid=self.uid,
             )
             if not (
                 promotion_data
@@ -59,8 +81,14 @@ class Publisher:
             ):
                 continue
 
+            check_in = (
+                datetime.fromisoformat(promotion_data.check_in)
+                if promotion_data.check_in
+                else None
+            )
             self.deployment_info_by_channel[channel] = DeploymentInfo(
                 success=promotion_data.success,
                 saas_file=promotion_data.saas_file,
                 target_config_hash=promotion_data.target_config_hash,
+                check_in=check_in,
             )

reconcile/saas_auto_promotions_manager/s3_exporter.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from enum import Enum
+
+from reconcile.saas_auto_promotions_manager.publisher import Publisher
+from reconcile.utils.state import State
+
+
+class DeploymentState(Enum):
+    SUCCESS = "success"
+    FAILED = "failed"
+    MISSING = "missing"
+
+
+@dataclass
+class PublisherData:
+    commit_sha: str
+    deployment_state: DeploymentState
+
+    @staticmethod
+    def from_publisher(publisher: Publisher) -> PublisherData:
+        """
+        Note, this data structure is a bit clumsy for this case, as for historic reasons we store deployment info
+        per channel and not per publisher.
+        We have a directed graph with n-n relationships, subscriber -> channel -> publisher.
+        This makes sense for the auto-promotion calculation, but can be misleading for the export,
+        as a publisher holds a deployment info per channel. We can consider the publisher deployment
+        state failed, if any of its channels failed.
+        """
+        deployment_state = DeploymentState.SUCCESS
+        failed = False
+        missing = False
+        for info in publisher.deployment_info_by_channel.values():
+            if not info:
+                missing = True
+            elif not info.success:
+                failed = True
+
+        if failed:
+            deployment_state = DeploymentState.FAILED
+        elif missing:
+            deployment_state = DeploymentState.MISSING
+
+        return PublisherData(
+            commit_sha=publisher.commit_sha,
+            deployment_state=deployment_state,
+        )
+
+
+class S3Exporter:
+    """
+    Export publisher deployment data to S3.
+    """
+
+    def __init__(self, state: State, dry_run: bool = True):
+        self._state = state
+        self._dry_run = dry_run
+
+    def export_publisher_data(self, publishers: Iterable[Publisher]) -> None:
+        data: dict[str, dict] = {}
+        for publisher in publishers:
+            publisher_data = PublisherData.from_publisher(publisher)
+            # Note, redeploy_on_config_change indicates that this publisher is a test job
+            key = f"{publisher.app_name}/{publisher.saas_name}/{publisher.resource_template_name}/{publisher.target_name}/{publisher.cluster_name}/{publisher.namespace_name}/{publisher.redeploy_on_config_change}"
+            data[key] = {
+                "commit_sha": publisher_data.commit_sha,
+                "deployment_state": publisher_data.deployment_state.value,
+            }
+
+        if not self._dry_run:
+            self._state.add(
+                key="publisher-data.json",
+                value=data,
+                force=True,
+            )