qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/saas_auto_promotions_manager/integration.py

@@ -1,23 +1,29 @@
 from collections.abc import Callable
-from typing import Optional
 
 from sretoolbox.utils import threaded
 
 from reconcile.openshift_saas_deploy import (
     QONTRACT_INTEGRATION as OPENSHIFT_SAAS_DEPLOY,
 )
-from reconcile.saas_auto_promotions_manager.merge_request_manager.merge_request_manager import (
-    MergeRequestManager,
+from reconcile.saas_auto_promotions_manager.merge_request_manager.batcher import (
+    Batcher,
+)
+from reconcile.saas_auto_promotions_manager.merge_request_manager.merge_request_manager_v2 import (
+    MergeRequestManagerV2,
+)
+from reconcile.saas_auto_promotions_manager.merge_request_manager.mr_parser import (
+    MRParser,
 )
 from reconcile.saas_auto_promotions_manager.merge_request_manager.renderer import (
     Renderer,
 )
+from reconcile.saas_auto_promotions_manager.meta import QONTRACT_INTEGRATION
 from reconcile.saas_auto_promotions_manager.publisher import Publisher
+from reconcile.saas_auto_promotions_manager.s3_exporter import S3Exporter
 from reconcile.saas_auto_promotions_manager.subscriber import Subscriber
 from reconcile.saas_auto_promotions_manager.utils.saas_files_inventory import (
     SaasFilesInventory,
 )
-from reconcile.saas_auto_promotions_manager.utils.vcs import VCS
 from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
 from reconcile.typed_queries.app_interface_vault_settings import (
     get_app_interface_vault_settings,
@@ -28,12 +34,9 @@ from reconcile.typed_queries.saas_files import get_saas_files
 from reconcile.utils.defer import defer
 from reconcile.utils.promotion_state import PromotionState
 from reconcile.utils.secret_reader import create_secret_reader
-from reconcile.utils.semver_helper import make_semver
-from reconcile.utils.state import init_state
+from reconcile.utils.state import State, init_state
 from reconcile.utils.unleash import get_feature_toggle_state
-
-QONTRACT_INTEGRATION = "saas-auto-promotions-manager"
-QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
+from reconcile.utils.vcs import VCS
 
 
 class SaasAutoPromotionsManager:
@@ -42,14 +45,16 @@ class SaasAutoPromotionsManager:
         deployment_state: PromotionState,
         vcs: VCS,
         saas_file_inventory: SaasFilesInventory,
-        merge_request_manager: MergeRequestManager,
+        merge_request_manager_v2: MergeRequestManagerV2,
+        s3_exporter: S3Exporter,
         thread_pool_size: int,
         dry_run: bool,
     ):
         self._deployment_state = deployment_state
         self._vcs = vcs
         self._saas_file_inventory = saas_file_inventory
-        self._merge_request_manager = merge_request_manager
+        self._merge_request_manager_v2 = merge_request_manager_v2
+        self._s3_exporter = s3_exporter
         self._thread_pool_size = thread_pool_size
         self._dry_run = dry_run
 
@@ -81,16 +86,25 @@ class SaasAutoPromotionsManager:
         self._fetch_publisher_real_world_states()
         self._compute_desired_subscriber_states()
         subscribers_with_diff = self._get_subscribers_with_diff()
-        self._merge_request_manager.fetch_sapm_managed_open_merge_requests()
-        self._merge_request_manager.housekeeping()
-        self._merge_request_manager.create_promotion_merge_requests(
-            subscribers=subscribers_with_diff
+        self._merge_request_manager_v2.reconcile(subscribers=subscribers_with_diff)
+        self._s3_exporter.export_publisher_data(
+            publishers=self._saas_file_inventory.publishers
         )
 
 
 def init_external_dependencies(
     dry_run: bool,
-) -> tuple[PromotionState, VCS, SaasFilesInventory, MergeRequestManager]:
+    env_name: str | None = None,
+    app_name: str | None = None,
+) -> tuple[
+    PromotionState,
+    VCS,
+    SaasFilesInventory,
+    MergeRequestManagerV2,
+    S3Exporter,
+    State,
+    State,
+]:
     """
     Lets initialize everything that involves calls to external dependencies:
     - VCS -> Gitlab / Github queries
@@ -117,38 +131,69 @@ def init_external_dependencies(
         allow_deleting_mrs=allow_deleting_mrs,
         allow_opening_mrs=allow_opening_mrs,
     )
-    merge_request_manager = MergeRequestManager(
+    mr_parser = MRParser(vcs=vcs)
+    merge_request_manager_v2 = MergeRequestManagerV2(
         vcs=vcs,
+        reconciler=Batcher(),
+        mr_parser=mr_parser,
         renderer=Renderer(),
     )
-    saas_files = get_saas_files()
+    saas_files = get_saas_files(env_name=env_name, app_name=app_name)
     saas_inventory = SaasFilesInventory(saas_files=saas_files)
+    saas_deploy_state = init_state(
+        integration=OPENSHIFT_SAAS_DEPLOY, secret_reader=secret_reader
+    )
     deployment_state = PromotionState(
-        state=init_state(integration=OPENSHIFT_SAAS_DEPLOY, secret_reader=secret_reader)
+        state=saas_deploy_state,
+    )
+    sapm_state = init_state(
+        integration=QONTRACT_INTEGRATION, secret_reader=secret_reader
+    )
+    s3_exporter = S3Exporter(
+        state=sapm_state,
+        dry_run=dry_run,
+    )
+    return (
+        deployment_state,
+        vcs,
+        saas_inventory,
+        merge_request_manager_v2,
+        s3_exporter,
+        saas_deploy_state,
+        sapm_state,
     )
-    return deployment_state, vcs, saas_inventory, merge_request_manager
 
 
 @defer
 def run(
     dry_run: bool,
     thread_pool_size: int,
-    defer: Optional[Callable] = None,
+    env_name: str | None = None,
+    app_name: str | None = None,
+    defer: Callable | None = None,
 ) -> None:
     (
         deployment_state,
         vcs,
         saas_inventory,
-        merge_request_manager,
-    ) = init_external_dependencies(dry_run=dry_run)
+        merge_request_manager_v2,
+        s3_exporter,
+        saas_deploy_state,
+        sapm_state,
+    ) = init_external_dependencies(
+        dry_run=dry_run, env_name=env_name, app_name=app_name
+    )
     if defer:
         defer(vcs.cleanup)
+        defer(saas_deploy_state.cleanup)
+        defer(sapm_state.cleanup)
 
     integration = SaasAutoPromotionsManager(
         deployment_state=deployment_state,
         vcs=vcs,
         saas_file_inventory=saas_inventory,
-        merge_request_manager=merge_request_manager,
+        merge_request_manager_v2=merge_request_manager_v2,
+        s3_exporter=s3_exporter,
         thread_pool_size=thread_pool_size,
         dry_run=dry_run,
     )

reconcile/saas_auto_promotions_manager/merge_request_manager/batcher.py

@@ -0,0 +1,208 @@
+from collections.abc import Iterable
+from dataclasses import dataclass
+from enum import Enum
+
+from reconcile.saas_auto_promotions_manager.merge_request_manager.open_merge_requests import (
+    OpenBatcherMergeRequest,
+)
+
+
+class Reason(Enum):
+    MISSING_UNBATCHING = "Closing this MR because it failed MR check and isn't marked as un-batchable yet."
+    OUTDATED_CONTENT = "Closing this MR because it has out-dated content."
+    NEW_BATCH = "Closing this MR in favor of a new batch MR."
+
+
+@dataclass(order=True)
+class Promotion:
+    content_hashes: set[str]
+    channels: set[str]
+
+
+@dataclass
+class Deletion:
+    mr: OpenBatcherMergeRequest
+    reason: Reason
+
+
+@dataclass
+class Addition:
+    content_hashes: set[str]
+    channels: set[str]
+    batchable: bool
+
+
+@dataclass
+class Diff:
+    deletions: list[Deletion]
+    additions: list[Addition]
+
+
+class Batcher:
+    """
+    The batcher calculates a Diff. I.e., which MRs need to be opened (Addition)
+    and which MRs need to be closed (Deletion). The batcher has no external
+    dependencies and does not interact with VCS. The batcher expects to be
+    given the desired state (which promotions do we want) and the current state
+    (the currently open MRs) in order to calculate the Diff.
+    """
+
+    def __init__(self) -> None:
+        self._desired_promotions: Iterable[Promotion] = []
+        self._open_mrs: Iterable[OpenBatcherMergeRequest] = []
+
+    def _unbatch(self, diff: Diff) -> None:
+        """
+        We optimistically batch MRs together that didnt run through MR check yet.
+        Vast majority of auto-promotion MRs are succeeding checks, so we can stay optimistic.
+        In the rare case of an MR failing the check, we want to unbatch it.
+        I.e., we open a dedicated MR for each channel in the batched MR, mark the new MRs as non-batchable
+        and close the old batched MR. By doing so, we ensure that unrelated MRs are not blocking each other.
+        Unbatched MRs are marked and will never be batched again.
+        """
+        open_mrs_after_unbatching: list[OpenBatcherMergeRequest] = []
+        unbatchable_hashes: set[str] = set()
+        falsely_marked_batchable_hashes: set[str] = set()
+        for mr in self._open_mrs:
+            if not mr.is_batchable:
+                unbatchable_hashes.update(mr.content_hashes)
+                open_mrs_after_unbatching.append(mr)
+            elif mr.failed_mr_check:
+                falsely_marked_batchable_hashes.update(mr.content_hashes)
+                diff.deletions.append(
+                    Deletion(
+                        mr=mr,
+                        reason=Reason.MISSING_UNBATCHING,
+                    )
+                )
+            else:
+                open_mrs_after_unbatching.append(mr)
+        self._open_mrs = open_mrs_after_unbatching
+
+        desired_promotions_after_unbatching: list[Promotion] = []
+        for promotion in self._desired_promotions:
+            if promotion.content_hashes.issubset(unbatchable_hashes):
+                desired_promotions_after_unbatching.append(promotion)
+                continue
+            elif promotion.content_hashes.issubset(falsely_marked_batchable_hashes):
+                diff.additions.append(
+                    Addition(
+                        content_hashes=promotion.content_hashes,
+                        channels=promotion.channels,
+                        batchable=False,
+                    )
+                )
+                continue
+            else:
+                desired_promotions_after_unbatching.append(promotion)
+        self._desired_promotions = desired_promotions_after_unbatching
+
+    def _remove_outdated(self, diff: Diff) -> None:
+        """
+        We want to be sure that the open MRs are still addressing desired content.
+        We close MRs that are not addressing any content hash in a desired promotion.
+        """
+        all_desired_content_hashes: set[str] = set()
+        for promotion in self._desired_promotions:
+            all_desired_content_hashes.update(promotion.content_hashes)
+
+        open_mrs_after_deletion: list[OpenBatcherMergeRequest] = []
+        for mr in self._open_mrs:
+            if mr.content_hashes.issubset(all_desired_content_hashes):
+                open_mrs_after_deletion.append(mr)
+                continue
+            diff.deletions.append(
+                Deletion(
+                    mr=mr,
+                    reason=Reason.OUTDATED_CONTENT,
+                )
+            )
+        self._open_mrs = open_mrs_after_deletion
+
+    def _batch_remaining_mrs(self, diff: Diff, batch_limit: int) -> None:
+        """
+        Remaining desired promotions should be batched together
+        if they are not addressed yet by a valid open MR.
+
+        We do not want to let MR batches grow infinitely,
+        as constant change of a batch might starve MRs that
+        are already part of the batch.
+        In order for MRs to not grow infinitely, we apply a
+        BATCH_LIMIT per MR, i.e., a maximum number of hashes
+        that can be batched together.
+        """
+        submitted_content_hashes: set[str] = set()
+        for open_mr in self._open_mrs:
+            submitted_content_hashes.update(open_mr.content_hashes)
+
+        unsubmitted_promotions = [
+            prom
+            for prom in self._desired_promotions
+            if not prom.content_hashes.issubset(submitted_content_hashes)
+        ]
+
+        if not unsubmitted_promotions:
+            return
+
+        batch_with_capacity: OpenBatcherMergeRequest | None = None
+        for mr in self._open_mrs:
+            if mr.is_batchable and len(mr.content_hashes) < batch_limit:
+                batch_with_capacity = mr
+                # Note, there should always only be maximum one batch with capacity available
+                break
+
+        if batch_with_capacity:
+            # We disassemble the batch to its promotions
+            # can be added to new batch(es)
+            unsubmitted_promotions.append(
+                Promotion(
+                    content_hashes=batch_with_capacity.content_hashes,
+                    channels=batch_with_capacity.channels,
+                )
+            )
+            # Lets close the current batch so remaining promotions can
+            # be aggregated in new batch(es)
+            diff.deletions.append(
+                Deletion(
+                    mr=batch_with_capacity,
+                    reason=Reason.NEW_BATCH,
+                )
+            )
+
+        batched_mr = Addition(
+            content_hashes=set(),
+            channels=set(),
+            batchable=True,
+        )
+
+        for promotion in unsubmitted_promotions:
+            batched_mr.content_hashes.update(promotion.content_hashes)
+            batched_mr.channels.update(promotion.channels)
+            if len(batched_mr.content_hashes) >= batch_limit:
+                # Note, we might also be above the batch limit, but thats ok.
+                # We only ensure that we create a new batch now and dont grow further.
+                diff.additions.append(batched_mr)
+                batched_mr = Addition(
+                    content_hashes=set(),
+                    channels=set(),
+                    batchable=True,
+                )
+        if batched_mr.content_hashes:
+            diff.additions.append(batched_mr)
+
+    def reconcile(
+        self,
+        desired_promotions: Iterable[Promotion],
+        open_mrs: Iterable[OpenBatcherMergeRequest],
+        batch_limit: int,
+    ) -> Diff:
+        self._open_mrs = open_mrs
+        self._desired_promotions = desired_promotions
+        diff = Diff(
+            deletions=[],
+            additions=[],
+        )
+        self._unbatch(diff=diff)
+        self._remove_outdated(diff=diff)
+        self._batch_remaining_mrs(diff=diff, batch_limit=batch_limit)
+        return diff

reconcile/saas_auto_promotions_manager/merge_request_manager/desired_state.py

@@ -0,0 +1,28 @@
+from collections import defaultdict
+from collections.abc import Iterable
+
+from reconcile.saas_auto_promotions_manager.merge_request_manager.batcher import (
+    Promotion,
+)
+from reconcile.saas_auto_promotions_manager.subscriber import Subscriber
+
+
+class DesiredState:
+    def __init__(self, subscribers: Iterable[Subscriber]) -> None:
+        self.content_hash_to_subscriber: dict[str, list[Subscriber]] = {}
+        subscribers_per_channel_combo: dict[str, list[Subscriber]] = defaultdict(list)
+        for subscriber in subscribers:
+            channel_combo = ",".join([c.name for c in subscriber.channels])
+            subscribers_per_channel_combo[channel_combo].append(subscriber)
+
+        desired_promotions: list[Promotion] = []
+        for channel_combo, subs in subscribers_per_channel_combo.items():
+            combined_content_hash = Subscriber.combined_content_hash(subscribers=subs)
+            self.content_hash_to_subscriber[combined_content_hash] = subs
+            desired_promotions.append(
+                Promotion(
+                    content_hashes={combined_content_hash},
+                    channels={channel_combo},
+                )
+            )
+        self.promotions = desired_promotions

reconcile/saas_auto_promotions_manager/merge_request_manager/merge_request.py

@@ -1,9 +1,8 @@
 import logging
-from collections.abc import Mapping
+from collections.abc import Iterable, Mapping
 
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.base import MergeRequestBase
-from reconcile.utils.mr.labels import AUTO_MERGE
 
 LOG = logging.getLogger(__name__)
 
@@ -23,13 +22,13 @@ class SAPMMR(MergeRequestBase):
         content_by_path: Mapping[str, str],
         description: str,
         title: str,
-        sapm_label: str,
+        labels: Iterable[str],
     ):
         super().__init__()
         self._content_by_path = content_by_path
         self._title = title
         self._description = description
-        self.labels = [AUTO_MERGE, sapm_label]
+        self.labels = labels
 
     @property
     def title(self) -> str:

reconcile/saas_auto_promotions_manager/merge_request_manager/merge_request_manager_v2.py

@@ -0,0 +1,172 @@
+import logging
+from collections.abc import Iterable
+
+from gitlab.exceptions import GitlabGetError
+
+from reconcile.change_owners.change_types import ChangeTypePriority
+from reconcile.saas_auto_promotions_manager.merge_request_manager.batcher import (
+    Addition,
+    Batcher,
+)
+from reconcile.saas_auto_promotions_manager.merge_request_manager.desired_state import (
+    DesiredState,
+)
+from reconcile.saas_auto_promotions_manager.merge_request_manager.merge_request import (
+    SAPMMR,
+)
+from reconcile.saas_auto_promotions_manager.merge_request_manager.metrics import (
+    SAPMClosedMRsCounter as MRClosedCounter,
+)
+from reconcile.saas_auto_promotions_manager.merge_request_manager.metrics import (
+    SAPMOpenedMRsCounter as MROpenedCounter,
+)
+from reconcile.saas_auto_promotions_manager.merge_request_manager.metrics import (
+    SAPMParallelOpenMRsGauge as ParallelOpenMRGauge,
+)
+from reconcile.saas_auto_promotions_manager.merge_request_manager.mr_parser import (
+    MRParser,
+)
+from reconcile.saas_auto_promotions_manager.merge_request_manager.renderer import (
+    Renderer,
+)
+from reconcile.saas_auto_promotions_manager.subscriber import Subscriber
+from reconcile.utils import metrics
+from reconcile.utils.mr.labels import prioritized_approval_label
+from reconcile.utils.vcs import VCS
+
+BATCH_SIZE_LIMIT = 5
+
+SAPM_LABEL = "SAPM"
+SAPM_MR_LABELS = [
+    SAPM_LABEL,
+    prioritized_approval_label(ChangeTypePriority.PROGRESSIVE_DELIVERY.value),
+]
+
+MR_DESC = """
+This is an auto-promotion triggered by app-interface's [saas-auto-promotions-manager](https://github.com/app-sre/qontract-reconcile/tree/master/reconcile/saas_auto_promotions_manager) (SAPM).
+This MR promotes all subscribers with auto-promotions for the channel(s) listed below.
+
+Please **do not manually modify** this MR in any way, as it might result in blocking the auto-promotion.
+
+This description is used by SAPM to manage auto-promotions.
+"""
+
+
+class MergeRequestManagerV2:
+    """
+    Manager for SAPM merge requests.
+
+    This class uses MRParser to fetch current state (currently open MRs).
+    This class calculates the desired state (i.e., desired promotions).
+    Desired state and current state are given to the Reconciler, which will
+    then determine a Diff (Additions and Deletions of MRs).
+
+    This class interacts with VCS to realize the result of the Diff.
+    """
+
+    def __init__(
+        self, vcs: VCS, mr_parser: MRParser, reconciler: Batcher, renderer: Renderer
+    ):
+        self._vcs = vcs
+        self._mr_parser = mr_parser
+        self._renderer = renderer
+        self._reconciler = reconciler
+        self._sapm_mrs: list[SAPMMR] = []
+
+    def _render_mr(self, addition: Addition) -> None:
+        subs: list[Subscriber] = []
+        for content_hash in addition.content_hashes:
+            subs.extend(self._desired_state.content_hash_to_subscriber[content_hash])
+        content_by_path: dict[str, str] = {}
+        has_error = False
+        for sub in subs:
+            if sub.target_file_path not in content_by_path:
+                try:
+                    content_by_path[sub.target_file_path] = (
+                        self._vcs.get_file_content_from_app_interface_master(
+                            file_path=sub.target_file_path
+                        )
+                    )
+                except GitlabGetError as e:
+                    if e.response_code == 404:
+                        logging.error(
+                            "The saas file %s does not exist anylonger. Most likely qontract-server data not in synch. This should resolve soon on its own.",
+                            sub.target_file_path,
+                        )
+                        has_error = True
+                        break
+                    raise e
+            content_by_path[sub.target_file_path] = (
+                self._renderer.render_merge_request_content(
+                    subscriber=sub,
+                    current_content=content_by_path[sub.target_file_path],
+                )
+            )
+        if has_error:
+            return
+
+        description_hashes = ",".join(addition.content_hashes)
+        description_channels = ",".join(addition.channels)
+
+        description = self._renderer.render_description(
+            message=MR_DESC,
+            content_hashes=description_hashes,
+            channels=description_channels,
+            is_batchable=addition.batchable,
+        )
+        title = self._renderer.render_title(
+            is_draft=False,
+            canary=False,
+            channels=description_channels,
+            batch_size=len(addition.content_hashes),
+        )
+        logging.info(
+            "Open MR for update in channel(s) %s",
+            description_channels,
+        )
+        self._sapm_mrs.append(
+            SAPMMR(
+                labels=SAPM_MR_LABELS,
+                content_by_path=content_by_path,
+                title=title,
+                description=description,
+            )
+        )
+
+    def reconcile(self, subscribers: Iterable[Subscriber]) -> None:
+        self._mr_parser.fetch_mrs(label=SAPM_LABEL)
+        current_state = self._mr_parser.get_open_batcher_mrs()
+        desired_state = DesiredState(subscribers=subscribers)
+        self._desired_state = desired_state
+
+        diff = self._reconciler.reconcile(
+            batch_limit=BATCH_SIZE_LIMIT,
+            desired_promotions=desired_state.promotions,
+            open_mrs=current_state,
+        )
+        parallel_open_mrs = (
+            len(current_state) - len(diff.deletions) + len(diff.additions)
+        )
+        metrics.set_gauge(ParallelOpenMRGauge(), parallel_open_mrs)
+        for deletion in diff.deletions:
+            metrics.inc_counter(
+                MRClosedCounter(
+                    reason=deletion.reason.name,
+                ),
+            )
+            self._vcs.close_app_interface_mr(
+                mr=deletion.mr.raw,
+                comment=deletion.reason.value,
+            )
+
+        for addition in diff.additions:
+            metrics.inc_counter(
+                MROpenedCounter(
+                    is_batchable=addition.batchable,
+                    batch_size=len(addition.content_hashes),
+                ),
+            )
+            self._render_mr(addition=addition)
+
+        for rendered_mr in self._sapm_mrs:
+            self._vcs.open_app_interface_merge_request(mr=rendered_mr)

reconcile/saas_auto_promotions_manager/merge_request_manager/metrics.py

@@ -0,0 +1,42 @@
+from pydantic import BaseModel
+
+from reconcile.utils.metrics import (
+    CounterMetric,
+    GaugeMetric,
+)
+
+
+class SAPMBaseMetric(BaseModel):
+    "Base class for SAPM MR metrics"
+
+    integration: str = "saas_auto_promotions_manager"
+
+
+class SAPMOpenedMRsCounter(SAPMBaseMetric, CounterMetric):
+    "Counter for the number of opened auto-promotion MRs"
+
+    is_batchable: bool
+    # We do not expect batches >10, i.e., cardinality will stay in check here.
+    batch_size: int
+
+    @classmethod
+    def name(cls) -> str:
+        return "sapm_opened_mrs"
+
+
+class SAPMClosedMRsCounter(SAPMBaseMetric, CounterMetric):
+    "Counter for the number of closed auto-promotion MRs"
+
+    reason: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "sapm_closed_mrs"
+
+
+class SAPMParallelOpenMRsGauge(SAPMBaseMetric, GaugeMetric):
+    "Gauge for the number of parallel open auto-promotion MRs"
+
+    @classmethod
+    def name(cls) -> str:
+        return "sapm_parallel_open_mrs"