qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/models.py

@@ -0,0 +1,123 @@
+from collections.abc import (
+    Callable,
+    Generator,
+    MutableMapping,
+)
+from typing import Any
+
+from croniter import croniter
+from pydantic import (
+    BaseModel,
+    ValidationError,
+)
+from pydantic import errors as pydantic_errors
+from pydantic.fields import ModelField
+
+DEFAULT_STRING = "I was too lazy to define a string here"
+DEFAULT_INT = 42
+
+
+def data_default_none(
+    klass: type[BaseModel], data: MutableMapping[str, Any], use_defaults: bool = True
+) -> MutableMapping[str, Any]:
+    """Set default values to None for required but optional fields."""
+    for field in klass.__fields__.values():
+        if not field.required:
+            continue
+
+        if field.alias not in data:
+            # Settings defaults
+            if field.allow_none:
+                data[field.alias] = None
+            elif not use_defaults:
+                raise ValueError(f"Field {field.alias} is required but not set.")
+            elif isinstance(field.type_, type) and issubclass(field.type_, str):
+                data[field.alias] = DEFAULT_STRING
+            elif isinstance(field.type_, type) and issubclass(field.type_, bool):
+                data[field.alias] = False
+            elif isinstance(field.type_, type) and issubclass(field.type_, int):
+                data[field.alias] = DEFAULT_INT
+        elif isinstance(field.type_, type) and issubclass(field.type_, BaseModel):
+            if isinstance(data[field.alias], dict):
+                data[field.alias] = data_default_none(field.type_, data[field.alias])
+            if isinstance(data[field.alias], list):
+                data[field.alias] = [
+                    data_default_none(field.type_, item)
+                    for item in data[field.alias]
+                    if isinstance(item, dict)
+                ]
+        elif field.sub_fields:
+            if all(
+                isinstance(sub_field.type_, type)
+                and issubclass(sub_field.type_, BaseModel)
+                for sub_field in field.sub_fields
+            ):
+                # Union[ClassA, ClassB] field
+                for sub_field in field.sub_fields:
+                    if isinstance(data[field.alias], dict):
+                        try:
+                            d = dict(data[field.alias])
+                            d.update(data_default_none(sub_field.type_, d))
+                            # Lets confirm we found a matching union class
+                            sub_field.type_(**d)
+                            data[field.alias] = d
+                            break
+                        except ValidationError:
+                            continue
+            elif isinstance(data[field.alias], list) and len(field.sub_fields) == 1:
+                # list[Union[ClassA, ClassB]] field
+                for sub_data in data[field.alias]:
+                    for sub_field in field.sub_fields[0].sub_fields or []:
+                        try:
+                            d = dict(sub_data)
+                            d.update(data_default_none(sub_field.type_, d))
+                            # Lets confirm we found a matching union class
+                            sub_field.type_(**d)
+                            sub_data.update(d)
+                            break
+                        except ValidationError:
+                            continue
+
+    return data
+
+
+class CSV(list[str]):
+    """
+    A pydantic custom type that converts a CSV into a list of strings. It
+    also supports basic validation of length constraints.
+    """
+
+    @classmethod
+    def __get_validators__(cls) -> Generator[Callable, None, None]:  # noqa: PLW3201
+        yield cls.validate
+        yield cls.length_validator
+
+    @classmethod
+    def validate(cls, value: str) -> list[str]:
+        items = [] if not value else value.split(",")
+        return items
+
+    @classmethod
+    def length_validator(
+        cls, v: "list[str]", values: dict, field: ModelField
+    ) -> "list[str]":
+        min_items = field.field_info.extra.get("csv_min_items")
+        max_items = field.field_info.extra.get("csv_max_items")
+
+        v_len = len(v)
+        if min_items is not None and v_len < min_items:
+            raise pydantic_errors.ListMinLengthError(limit_value=min_items)
+        if max_items is not None and v_len > max_items:
+            raise pydantic_errors.ListMaxLengthError(limit_value=max_items)
+        return v
+
+
+def cron_validator(value: str) -> str:
+    """
+    A pydantic validator for a cron expression.
+    """
+    try:
+        croniter(value)
+        return value
+    except ValueError as e:
+        raise ValueError(f"Invalid cron expression: {e}") from None

reconcile/utils/mr/README.md

@@ -0,0 +1,198 @@
+# Merge Requests
+
+Module responsible for abstracting a Gitlab Merge Request.
+
+## Creating New MergeRequest Classes
+
+Each type of merge request requires a class that inherits
+from reconcile.utils.mr.base.MergeRequestBase.
+
+That class has to comply with the specification below.
+
+### Class Name
+
+The class name is given by the class variable `name`. That name will be
+used all around, the most important place being the SQS Message parameter
+`pr_type`, used later to create an instance of the same class.
+
+While the name can be anything it is mandatory that it is defined.
+
+### Class Initialization
+
+The `__init__` method has to call `super().__init__()` right after the minimum
+parameters initialization. That is required to compose the SQS Message that
+will later be used to create a new instance of the same class.
+
+### Class Methods
+
+The minimum methods that have to be defined are:
+
+* `title`: a property that is used to give the Gitlab Merge Request a title.
+  It can also be used for building commit messages or as content to committed
+  files.
+* `description`: a description for the Merge Request.
+* `process`: this method is called when submitting the Merge Request to gitlab.
+  it's the place for the merge request changes, like creating, updating and
+  deleting files. The `process` method is called after the local branch is
+  created and right after it the `gitlab_cli.project.mergerequests.create()`
+  is called.
+
+### Example
+
+This is an example of a minimum implementation for a new MergeRequest class:
+
+```python
+from reconcile.utils.mr.base import MergeRequestBase
+
+
+class CreateDeleteUser(MergeRequestBase):
+
+    name = 'create_delete_user_mr'
+
+    def __init__(self, username, paths):
+        self.username = username
+        self.paths = paths
+
+        # Called right after the minimum parameters to recreate
+        # an instance with the same data. This is important for
+        # building up the SQS message payload.
+        super().__init__()
+
+    @property
+    def title(self) -> str:
+        return f'[{self.name}] delete user {self.username}'
+
+    @property
+    def description(self) -> str:
+        return f'delete user {self.username}'
+
+    def process(self, gitlab_cli):
+        for path in self.paths:
+            gitlab_cli.delete_file(branch_name=self.branch,
+                                   file_path=path,
+                                   commit_message=self.title)
+```
+
+## Sending MRs to SQS
+
+To send a Merge Request to SQS, create the corresponding MergeRequest object:
+
+```python
+from reconcile.utils.mr import CreateAppInterfaceNotificator
+
+
+notification = {
+    'notification_type': 'Outage',
+    'description': 'The AppSRE team is currently investigating an outage',
+    'short_description': 'Short Description',
+}
+
+merge_request = CreateAppInterfaceNotificator(notification=notification)
+
+```
+
+then create the SQS Client instance:
+
+```python
+from reconcile import queries
+
+from reconcile.utils.sqs_gateway import SQSGateway
+from reconcile.utils.secret_reader import SecretReader
+
+
+accounts = queries.get_queue_aws_accounts()
+secretReader = SecretReader(queries.get_secret_reader_settings())
+sqs_cli = SQSGateway(accounts, secret_reader=secret_reader)
+```
+
+and then submit the merge request to the SQS:
+
+```python
+merge_request.submit_to_sqs(sqs_cli=sqs_cli)
+```
+
+## Sending MRs to Gitlab
+
+To get the message from SQS and use it for sending a Merge Request to gitlab,
+first get the SQS messages:
+
+
+```python
+from reconcile import queries
+
+from reconcile.utils.sqs_gateway import SQSGateway
+from reconcile.utils.secret_reader import SecretReader
+
+
+accounts = queries.get_queue_aws_accounts()
+settings = queries.get_app_interface_settings()
+
+secretReader = SecretReader(queries.get_secret_reader_settings())
+sqs_cli = SQSGateway(accounts, secret_reader=secret_reader)
+messages = sqs_cli.receive_messages()
+```
+
+then create the Gitlab client instance:
+
+```python
+from reconcile.utils.gitlab_api import GitLabApi
+
+instance = queries.get_gitlab_instance()
+saas_files = queries.get_saas_files_minimal()
+gitlab_cli = GitLabApi(instance, project_id=gitlab_project_id,
+                       settings=settings)
+```
+
+and then loop the messages, creating the MergeRequest objects and submitting
+the merge requests:
+
+```python
+from reconcile.utils.mr import init_from_sqs_message
+
+for message in messages:
+    receipt_handle, body = message[0], message[1]
+    merge_request = init_from_sqs_message(body)
+    merge_request.submit_to_gitlab(gitlab_cli=gitlab_cli)
+    sqs_cli.delete_message(receipt_handle)
+```
+
+## Using the MR Module for Qontract-reconcile Integrations
+
+A given integration might be executed in different environments, like:
+
+* Developer local machine.
+* OpenShift cluster outside the VPN.
+* OpenShift cluster inside the VPN.
+* Jenkins periodic job.
+
+Because we don't want the integrations to care if they are running inside or
+outside the VPN, we use the `reconcile.mr_client_gateway.init()` to get the
+on of SQS or GitLab client, according to the App Interface settings. Example:
+
+```python
+from reconcile import mr_client_gateway
+from reconcile.utils.mr import CreateDeleteAwsAccessKey
+
+
+def run(dry_run, gitlab_project_id):
+    ...
+    mr = CreateDeleteAwsAccessKey(...)
+    with mr_client_gateway.init(gitlab_project_id=gitlab_project_id) as mr_cli:
+      mr.submit(cli=mr_cli)
+```
+
+If we want to override what's set in App Interface and get a specific client,
+say `gitlab`, we would:
+
+```python
+from reconcile import mr_client_gateway
+from reconcile.utils.mr import CreateDeleteAwsAccessKey
+
+
+def run(dry_run, gitlab_project_id):
+    ...
+    mr = CreateDeleteAwsAccessKey(...)
+    with mr_client_gateway.init(sqs_or_gitlab='gitlab',
+                           gitlab_project_id=gitlab_project_id) as mr_cli:
+      mr.submit(cli=mr_cli)
+```

reconcile/utils/mr/__init__.py

@@ -1,5 +1,4 @@
 from reconcile.utils.mr.app_interface_reporter import CreateAppInterfaceReporter
-from reconcile.utils.mr.auto_promoter import AutoPromoter
 from reconcile.utils.mr.aws_access import CreateDeleteAwsAccessKey
 from reconcile.utils.mr.base import (
     MergeRequestBase,
@@ -13,24 +12,29 @@ from reconcile.utils.mr.ocm_update_recommended_version import (
 from reconcile.utils.mr.ocm_upgrade_scheduler_org_updates import (
     CreateOCMUpgradeSchedulerOrgUpdates,
 )
+from reconcile.utils.mr.promote_qontract import (
+    PromoteQontractReconcileCommercial,
+    PromoteQontractSchemas,
+)
 from reconcile.utils.mr.user_maintenance import (
     CreateDeleteUserAppInterface,
     CreateDeleteUserInfra,
 )
 
 __all__ = [
-    "init_from_sqs_message",
-    "UnknownMergeRequestType",
-    "MergeRequestProcessingError",
+    "CreateAppInterfaceNotificator",
     "CreateAppInterfaceReporter",
-    "CreateDeleteAwsAccessKey",
     "CreateClustersUpdates",
-    "CreateOCMUpgradeSchedulerOrgUpdates",
-    "CreateOCMUpdateRecommendedVersion",
-    "CreateAppInterfaceNotificator",
+    "CreateDeleteAwsAccessKey",
     "CreateDeleteUserAppInterface",
     "CreateDeleteUserInfra",
-    "AutoPromoter",
+    "CreateOCMUpdateRecommendedVersion",
+    "CreateOCMUpgradeSchedulerOrgUpdates",
+    "MergeRequestProcessingError",
+    "PromoteQontractReconcileCommercial",
+    "PromoteQontractSchemas",
+    "UnknownMergeRequestType",
+    "init_from_sqs_message",
 ]
 
 
@@ -40,7 +44,7 @@ class UnknownMergeRequestType(Exception):
     """
 
 
-def init_from_sqs_message(message):
+def init_from_sqs_message(message) -> MergeRequestBase:
     # First, let's find the classes that are inheriting from
     # MergeRequestBase and create a map where the class.name is
     # the key and the class itself is the value.

reconcile/utils/mr/app_interface_reporter.py

@@ -3,6 +3,7 @@ from pathlib import Path
 
 from ruamel.yaml.scalarstring import PreservedScalarString as pss
 
+from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.base import (
     MergeRequestBase,
     app_interface_email,
@@ -11,7 +12,6 @@ from reconcile.utils.mr.labels import AUTO_MERGE
 
 
 class CreateAppInterfaceReporter(MergeRequestBase):
-
     name = "create_app_interface_reporter_mr"
 
     def __init__(self, reports, email_body, reports_path):
@@ -35,7 +35,7 @@ class CreateAppInterfaceReporter(MergeRequestBase):
     def description(self) -> str:
         return f"reports for {self.isodate}"
 
-    def process(self, gitlab_cli):
+    def process(self, gitlab_cli: GitLabApi) -> None:
         actions = [
             {
                 "action": "create",

reconcile/utils/mr/aws_access.py

@@ -5,6 +5,7 @@ from ruamel import yaml
 from ruamel.yaml.scalarstring import PreservedScalarString as pss
 
 from reconcile.utils.constants import PROJ_ROOT
+from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.base import (
     MergeRequestBase,
     app_interface_email,
@@ -15,7 +16,6 @@ BODY_TEMPLATE = PROJ_ROOT / "templates" / "aws_access_key_email.j2"
 
 
 class CreateDeleteAwsAccessKey(MergeRequestBase):
-
     name = "create_delete_aws_access_key_mr"
 
     def __init__(self, account, path, key):
@@ -35,10 +35,10 @@ class CreateDeleteAwsAccessKey(MergeRequestBase):
     def description(self) -> str:
         return f"delete {self.account} access key {self.key}"
 
-    def process(self, gitlab_cli):
+    def process(self, gitlab_cli: GitLabApi) -> None:
         # add key to deleteKeys list to be picked up by aws-iam-keys
         raw_file = gitlab_cli.project.files.get(
-            file_path=self.path, ref=self.main_branch
+            file_path=self.path, ref=gitlab_cli.main_branch
         )
         content = yaml.load(raw_file.decode(), Loader=yaml.RoundTripLoader)
 
@@ -57,7 +57,7 @@ class CreateDeleteAwsAccessKey(MergeRequestBase):
         )
 
         # add a new email to be picked up by email-sender
-        with open(BODY_TEMPLATE) as file_obj:
+        with open(BODY_TEMPLATE, encoding="locale") as file_obj:
             body_template = Template(
                 file_obj.read(), keep_trailing_newline=True, trim_blocks=True
             )

reconcile/utils/mr/base.py

@@ -1,18 +1,18 @@
 import json
 import logging
 from abc import (
-    ABCMeta,
+    ABC,
     abstractmethod,
 )
-from typing import (
-    Optional,
-    Union,
-)
+from collections.abc import Iterable
+from typing import Any
 from uuid import uuid4
 
 from gitlab.exceptions import GitlabError
 from jinja2 import Template
 
+from reconcile import typed_queries
+from reconcile.gql_definitions.fragments.user import User
 from reconcile.utils.constants import PROJ_ROOT
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.labels import DO_NOT_MERGE_HOLD
@@ -35,10 +35,10 @@ class MergeRequestProcessingError(Exception):
     """
 
 
-MRClient = Union[GitLabApi, SQSGateway]
+MRClient = GitLabApi | SQSGateway
 
 
-class MergeRequestBase(metaclass=ABCMeta):
+class MergeRequestBase(ABC):
     """
     Base abstract class for all merge request types.
     """
@@ -52,19 +52,17 @@ class MergeRequestBase(metaclass=ABCMeta):
         # of the child class.
         self.sqs_msg_data = {**self.__dict__}
 
-        self.gitlab_cli = None
         self.labels = [DO_NOT_MERGE_HOLD]
 
         random_id = str(uuid4())[:6]
         self.branch = f"{self.name}-{random_id}"
         self.branch_created = False
 
-        self.main_branch = "master"
         self.remove_source_branch = True
 
         self.cancelled = False
 
-    def cancel(self, message):
+    def cancel(self, message: str) -> None:
         self.cancelled = True
         raise CancelMergeRequest(
             f"{self.name} MR canceled for "
@@ -93,7 +91,7 @@ class MergeRequestBase(metaclass=ABCMeta):
         """
 
     @abstractmethod
-    def process(self, gitlab_cli):
+    def process(self, gitlab_cli: GitLabApi) -> None:
         """
         Called by `submit_to_gitlab`, this method is the place for
         user-defined steps to create the commits of a merge request.
@@ -103,7 +101,7 @@ class MergeRequestBase(metaclass=ABCMeta):
         """
 
     @property
-    def sqs_data(self):
+    def sqs_data(self) -> dict[str, Any]:
         """
         The SQS Message payload (MessageBody) generated out of
         the Merge Request class instance.
@@ -113,7 +111,27 @@ class MergeRequestBase(metaclass=ABCMeta):
             **self.sqs_msg_data,
         }
 
-    def submit_to_sqs(self, sqs_cli) -> None:
+    def infer_author(
+        self, author_email: str | None, all_users: Iterable[User] | None = None
+    ) -> str | None:
+        if not author_email:
+            return None
+        if not all_users:
+            return None
+
+        username = author_email.split("@")[0]
+        users = None
+        if author_email.endswith(typed_queries.smtp.settings().mail_address):
+            users = [u for u in all_users if username == u.org_username]
+        elif author_email.endswith("users.noreply.github.com"):
+            users = [u for u in all_users if username == u.github_username]
+
+        if users:
+            return users[0].org_username
+
+        return None
+
+    def submit_to_sqs(self, sqs_cli: SQSGateway) -> None:
         """
         Sends the MR message to SQS.
 
@@ -122,21 +140,20 @@ class MergeRequestBase(metaclass=ABCMeta):
         """
         sqs_cli.send_message(self.sqs_data)
 
-    @property
-    def gitlab_data(self):
+    def gitlab_data(self, target_branch: str) -> dict[str, Any]:
         """
         The Gitlab payload for creating the Merge Request.
         """
         return {
             "source_branch": self.branch,
-            "target_branch": self.main_branch,
+            "target_branch": target_branch,
             "title": self.title,
             "description": self.description,
             "remove_source_branch": self.remove_source_branch,
             "labels": self.labels,
         }
 
-    def submit_to_gitlab(self, gitlab_cli):
+    def submit_to_gitlab(self, gitlab_cli: GitLabApi) -> Any:
         """
         Sends the MR to Gitlab.
 
@@ -163,11 +180,13 @@ class MergeRequestBase(metaclass=ABCMeta):
             # Avoiding empty MRs
             if not self.diffs(gitlab_cli):
                 self.cancel(
-                    f"No changes when compared to {self.main_branch}. "
+                    f"No changes when compared to {gitlab_cli.main_branch}. "
                     "Aborting MR creation."
                 )
 
-            return gitlab_cli.project.mergerequests.create(self.gitlab_data)
+            return gitlab_cli.project.mergerequests.create(
+                self.gitlab_data(target_branch=gitlab_cli.main_branch)
+            )
         except CancelMergeRequest as mr_cancel:
             # cancellation is a valid behaviour. it indicates, that the
             # operation is not required, therefore we will not signal
@@ -187,14 +206,14 @@ class MergeRequestBase(metaclass=ABCMeta):
                 f"Reason: {err}"
             ) from err
 
-    def ensure_tmp_branch_exists(self, gitlab_cli):
+    def ensure_tmp_branch_exists(self, gitlab_cli: GitLabApi) -> None:
         if not self.branch_created:
             gitlab_cli.create_branch(
-                new_branch=self.branch, source_branch=self.main_branch
+                new_branch=self.branch, source_branch=gitlab_cli.main_branch
             )
             self.branch_created = True
 
-    def delete_tmp_branch(self, gitlab_cli):
+    def delete_tmp_branch(self, gitlab_cli: GitLabApi) -> None:
         if self.branch_created:
             try:
                 gitlab_cli.delete_branch(branch=self.branch)
@@ -206,17 +225,18 @@ class MergeRequestBase(metaclass=ABCMeta):
                     f"Failed to delete branch {self.branch}. " f"Reason: {gitlab_error}"
                 )
 
-    def diffs(self, gitlab_cli):
+    def diffs(self, gitlab_cli: GitLabApi) -> Any:
         return gitlab_cli.project.repository_compare(
-            from_=self.main_branch, to=self.branch
+            from_=gitlab_cli.main_branch, to=self.branch
         )["diffs"]
 
-    def submit(self, cli: MRClient):
+    def submit(self, cli: MRClient) -> Any | None:
         if isinstance(cli, GitLabApi):
             return self.submit_to_gitlab(gitlab_cli=cli)
 
         if isinstance(cli, SQSGateway):
-            return self.submit_to_sqs(sqs_cli=cli)
+            self.submit_to_sqs(sqs_cli=cli)
+            return None
 
         raise AttributeError(f"client {cli} not supported")
 
@@ -225,13 +245,13 @@ def app_interface_email(
     name: str,
     subject: str,
     body: str,
-    users: Optional[list[str]] = None,
-    aliases: Optional[list[str]] = None,
-    aws_accounts: Optional[list[str]] = None,
-    apps: Optional[list[str]] = None,
+    users: list[str] | None = None,
+    aliases: list[str] | None = None,
+    aws_accounts: list[str] | None = None,
+    apps: list[str] | None = None,
 ) -> str:
     """Render app-interface-email template."""
-    with open(EMAIL_TEMPLATE) as file_obj:
+    with open(EMAIL_TEMPLATE, encoding="locale") as file_obj:
         email_template = Template(
             file_obj.read(), keep_trailing_newline=True, trim_blocks=True
         )

reconcile/utils/mr/clusters_updates.py

@@ -3,14 +3,16 @@ from io import StringIO
 from ruamel.yaml import YAML
 
 from reconcile.change_owners.decision import DecisionCommand
+from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.base import MergeRequestBase
 
 yaml = YAML()
-yaml.explicit_start = True  # type: ignore
+yaml.explicit_start = True
+# Lets prevent line wraps
+yaml.width = 4096
 
 
 class CreateClustersUpdates(MergeRequestBase):
-
     name = "create_clusters_updates_mr"
 
     def __init__(self, clusters_updates):
@@ -28,7 +30,7 @@ class CreateClustersUpdates(MergeRequestBase):
     def description(self) -> str:
         return DecisionCommand.APPROVED.value
 
-    def process(self, gitlab_cli):
+    def process(self, gitlab_cli: GitLabApi):
         changes = False
         for cluster_name, cluster_updates in self.clusters_updates.items():
             if not cluster_updates:
@@ -36,7 +38,7 @@ class CreateClustersUpdates(MergeRequestBase):
 
             cluster_path = cluster_updates.pop("path")
             raw_file = gitlab_cli.project.files.get(
-                file_path=cluster_path, ref=self.main_branch
+                file_path=cluster_path, ref=gitlab_cli.main_branch
             )
             content = yaml.load(raw_file.decode())
             if "spec" not in content:
@@ -55,15 +57,16 @@ class CreateClustersUpdates(MergeRequestBase):
             # content.update(cluster_updates) :(
             content.update(cluster_updates["root"])
 
-            new_content = StringIO()
-            yaml.dump(content, new_content)
+            with StringIO() as stream:
+                yaml.dump(content, stream)
+                new_content = stream.getvalue()
 
             msg = f"update cluster {cluster_name} spec fields"
             gitlab_cli.update_file(
                 branch_name=self.branch,
                 file_path=cluster_path,
                 commit_message=msg,
-                content=new_content.getvalue(),
+                content=new_content,
             )
 
         if not changes: