qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/secret_reader.py

@@ -5,13 +5,11 @@ from abc import (
 from collections.abc import Mapping
 from typing import (
     Any,
-    Optional,
     Protocol,
-    Union,
+    runtime_checkable,
 )
 
 from hvac.exceptions import Forbidden
-from sretoolbox.utils import retry
 
 from reconcile.utils import (
     config,
@@ -28,6 +26,7 @@ class SecretNotFound(Exception):
     pass
 
 
+@runtime_checkable
 class HasSecret(Protocol):
     """HasSecret defines all attributes needed to fetch a secret from Vault or config.
 
@@ -36,20 +35,20 @@ class HasSecret(Protocol):
 
     path: str
     field: str
-    version: Optional[int]
-    q_format: Optional[str]
+    version: int | None
+    q_format: str | None
 
 
 class SecretReaderBase(ABC):
     @abstractmethod
     def _read(
-        self, path: str, field: str, format: Optional[str], version: Optional[int]
+        self, path: str, field: str, format: str | None, version: int | None
     ) -> str:
         raise NotImplementedError()
 
     @abstractmethod
     def _read_all(
-        self, path: str, field: str, format: Optional[str], version: Optional[int]
+        self, path: str, field: str, format: str | None, version: int | None
     ) -> dict[str, str]:
         raise NotImplementedError()
 
@@ -96,7 +95,7 @@ class SecretReaderBase(ABC):
         )
 
     def read_with_parameters(
-        self, path: str, field: str, format: Optional[str], version: Optional[int]
+        self, path: str, field: str, format: str | None, version: int | None
     ) -> str:
         return self._read(
             path=path,
@@ -106,7 +105,7 @@ class SecretReaderBase(ABC):
         )
 
     def read_all_with_parameters(
-        self, path: str, field: str, format: Optional[str], version: Optional[int]
+        self, path: str, field: str, format: str | None, version: int | None
     ) -> dict[str, str]:
         return self._read_all(
             path=path,
@@ -116,7 +115,7 @@ class SecretReaderBase(ABC):
         )
 
     def _parameters_to_dict(
-        self, path: str, field: str, format: Optional[str], version: Optional[int]
+        self, path: str, field: str, format: str | None, version: int | None
     ) -> dict[str, Any]:
         return {
             "path": path,
@@ -126,7 +125,7 @@ class SecretReaderBase(ABC):
         }
 
     @staticmethod
-    def to_dict(secret: HasSecret) -> dict[str, Union[str, int, None]]:
+    def to_dict(secret: HasSecret) -> dict[str, str | int | None]:
         """Convenient method to convert Secret class to dictionary."""
         return {
             "path": secret.path,
@@ -141,7 +140,7 @@ class VaultSecretReader(SecretReaderBase):
     Read secrets from vault via a vault_client
     """
 
-    def __init__(self, vault_client: Optional[VaultClient] = None):
+    def __init__(self, vault_client: VaultClient | None = None):
         self._vault_client = vault_client
 
     @property
@@ -150,9 +149,8 @@ class VaultSecretReader(SecretReaderBase):
             self._vault_client = VaultClient()
         return self._vault_client
 
-    @retry()
     def _read_all(
-        self, path: str, field: str, format: Optional[str], version: Optional[int]
+        self, path: str, field: str, format: str | None, version: int | None
     ) -> dict[str, str]:
         try:
             data = self.vault_client.read_all(  # type: ignore[attr-defined] # mypy doesn't recognize the VaultClient.__new__ method
@@ -166,15 +164,14 @@ class VaultSecretReader(SecretReaderBase):
         except Forbidden:
             raise VaultForbidden(
                 f"permission denied reading vault secret " f"at {path}"
-            )
+            ) from None
         except vault.SecretNotFound as e:
             raise SecretNotFound(*e.args) from e
         return data
 
-    @retry()
     def _read(
-        self, path: str, field: str, format: Optional[str], version: Optional[int]
-    ) -> dict[str, str]:
+        self, path: str, field: str, format: str | None, version: int | None
+    ) -> str:
         try:
             data = self.vault_client.read(  # type: ignore[attr-defined] # mypy doesn't recognize the VaultClient.__new__ method
                 self._parameters_to_dict(
@@ -195,7 +192,7 @@ class ConfigSecretReader(SecretReaderBase):
     """
 
     def _read(
-        self, path: str, field: str, format: Optional[str], version: Optional[int]
+        self, path: str, field: str, format: str | None, version: int | None
     ) -> str:
         try:
             data = config.read(
@@ -211,7 +208,7 @@ class ConfigSecretReader(SecretReaderBase):
         return data
 
     def _read_all(
-        self, path: str, field: str, format: Optional[str], version: Optional[int]
+        self, path: str, field: str, format: str | None, version: int | None
     ) -> dict[str, str]:
         try:
             data = config.read_all(
@@ -243,13 +240,13 @@ class SecretReader(SecretReaderBase):
     Consider using create_secret_reader() instead.
     """
 
-    def __init__(self, settings: Optional[Mapping] = None) -> None:
+    def __init__(self, settings: Mapping | None = None) -> None:
         """
         :param settings: app-interface-settings object. It is a dictionary
         containing `value: true` if Vault is to be used as the secret backend.
         """
         self.settings = settings
-        self._vault_client: Optional[VaultClient] = None
+        self._vault_client: VaultClient | None = None
 
     @property
     def vault_client(self) -> VaultClient:
@@ -257,9 +254,8 @@ class SecretReader(SecretReaderBase):
             self._vault_client = VaultClient()
         return self._vault_client
 
-    @retry()
     def _read(
-        self, path: str, field: str, format: Optional[str], version: Optional[int]
+        self, path: str, field: str, format: str | None, version: int | None
     ) -> str:
         """Returns a value of a key from Vault secret or configuration file.
         The input secret is a dictionary which contains the following fields:
@@ -293,9 +289,8 @@ class SecretReader(SecretReaderBase):
 
         return data
 
-    @retry()
     def _read_all(
-        self, path: str, field: str, format: Optional[str], version: Optional[int]
+        self, path: str, field: str, format: str | None, version: int | None
     ) -> dict[str, str]:
         """Returns a dictionary of keys and values
         from Vault secret or configuration file.
@@ -321,7 +316,7 @@ class SecretReader(SecretReaderBase):
             except Forbidden:
                 raise VaultForbidden(
                     f"permission denied reading vault secret " f"at {path}"
-                )
+                ) from None
             except vault.SecretNotFound as e:
                 raise SecretNotFound(*e.args) from e
         else:

reconcile/utils/semver_helper.py

@@ -7,7 +7,16 @@ def make_semver(major: int, minor: int, patch: int) -> str:
     return str(semver.VersionInfo(major=major, minor=minor, patch=patch))
 
 
-def parse_semver(version: str) -> semver.VersionInfo:
+def parse_semver(
+    version: str, optional_minor_and_patch: bool = False
+) -> semver.VersionInfo:
+    if optional_minor_and_patch:
+        # semver3 supports optional minor and patch.
+        # until we upgrade to semver3, we support this by adding a default minor and/or patch
+        if "." not in version:
+            version = f"{version}.0.0"
+        elif version.count(".") == 1:
+            version = f"{version}.0"
     return semver.VersionInfo.parse(version)
 
 
@@ -26,3 +35,8 @@ def sort_versions(versions: Iterable[str]) -> list[str]:
 
 def is_version_bumped(current_version: str, previous_version: str) -> bool:
     return parse_semver(current_version) > parse_semver(previous_version)
+
+
+def get_version_prefix(version: str) -> str:
+    semver = parse_semver(version)
+    return f"{semver.major}.{semver.minor}"

reconcile/utils/slack_api.py

@@ -9,9 +9,7 @@ from collections.abc import (
 )
 from typing import (
     Any,
-    Optional,
     Protocol,
-    Union,
 )
 
 from slack_sdk import WebClient
@@ -24,6 +22,8 @@ from slack_sdk.http_retry import (
     RetryState,
 )
 
+from reconcile.utils.metrics import slack_request
+
 MAX_RETRIES = 5
 TIMEOUT = 30
 
@@ -44,36 +44,32 @@ class ServerErrorRetryHandler(RetryHandler):
         *,
         state: RetryState,
         request: HttpRequest,
-        response: Optional[HttpResponse] = None,
-        error: Optional[Exception] = None,
+        response: HttpResponse | None = None,
+        error: Exception | None = None,
     ) -> bool:
         return response is not None and response.status_code >= 500
 
 
 class HasClientGlobalConfig(Protocol):
-    max_retries: Optional[int]
-    timeout: Optional[int]
+    max_retries: int | None
+    timeout: int | None
 
-    def dict(self) -> dict[str, Optional[int]]:
-        ...
+    def dict(self) -> dict[str, int | None]: ...
 
 
 class HasClientMethodConfig(Protocol):
     name: str
     args: Any
 
-    def dict(self) -> dict[str, str]:
-        ...
+    def dict(self) -> dict[str, str]: ...
 
 
 class HasClientConfig(Protocol):
     @property
-    def q_global(self) -> Optional[HasClientGlobalConfig]:
-        ...
+    def q_global(self) -> HasClientGlobalConfig | None: ...
 
     @property
-    def methods(self) -> Optional[Sequence[HasClientMethodConfig]]:
-        ...
+    def methods(self) -> Sequence[HasClientMethodConfig] | None: ...
 
 
 class SlackApiConfig:
@@ -83,7 +79,6 @@ class SlackApiConfig:
     """
 
     def __init__(self, timeout: int = TIMEOUT, max_retries: int = MAX_RETRIES) -> None:
-
         self.timeout = timeout
         self.max_retries = max_retries
         self._methods: dict[str, Any] = {}
@@ -98,7 +93,7 @@ class SlackApiConfig:
         """
         self._methods[method_name] = method_config
 
-    def get_method_config(self, method_name: str) -> Optional[dict[str, Any]]:
+    def get_method_config(self, method_name: str) -> dict[str, Any] | None:
         """
         Get Slack method configuration.
         :param method_name: the name of a method (ex. users.list)
@@ -168,9 +163,10 @@ class SlackApi:
         self,
         workspace_name: str,
         token: str,
-        api_config: Optional[SlackApiConfig] = None,
+        api_config: SlackApiConfig | None = None,
         init_usergroups: bool = True,
-        channel: Optional[str] = None,
+        channel: str | None = None,
+        slack_url: str | None = None,
         **chat_kwargs: Any,
     ) -> None:
         """
@@ -181,7 +177,7 @@ class SlackApi:
         :param init_usergroups: whether or not to get a list of all Slack
         usergroups when instantiated
         :param channel: the Slack channel to post messages to, only used
-        when posting messages to a channel
+        when posting messages to a channel or getting conversation history
         :param chat_kwargs: any other kwargs that can be used to post Slack
         channel messages
         """
@@ -192,14 +188,22 @@ class SlackApi:
         else:
             self.config = SlackApiConfig()
 
-        self._sc = WebClient(token=token, timeout=self.config.timeout)
+        self._sc = WebClient(
+            token=token,
+            timeout=self.config.timeout,
+            base_url=slack_url or WebClient.BASE_URL,
+        )
         self._configure_client_retry()
 
         self._results: dict[str, Any] = {}
+        self._enterprise_user_id_to_user_ids: dict[str, str] = {}
 
         self.channel = channel
         self.chat_kwargs = chat_kwargs
 
+        self._user_groups_initialized = False
+        self.usergroups: list[dict] = []
+
         if init_usergroups:
             self._initiate_usergroups()
 
@@ -234,16 +238,25 @@ class SlackApi:
             )
 
         def do_send(c: str, t: str) -> None:
+            slack_request.labels("chat.postMessage", "POST").inc()
             self._sc.chat_postMessage(channel=c, text=t, **self.chat_kwargs)
 
         try:
             do_send(self.channel, text)
         except SlackApiError as e:
-            if e.response["error"] == "not_in_channel":
-                self.join_channel()
-                do_send(self.channel, text)
-            else:
-                raise e
+            match e.response["error"]:
+                case "not_in_channel":
+                    self.join_channel()
+                    do_send(self.channel, text)
+                # When a message is sent to #someChannel and the Slack API can't find
+                # it, the message it provides in the exception doesn't include the
+                # channel name. We handle that here in case the consumer has many such
+                # independent calls to Slack's API.
+                case "channel_not_found":
+                    logging.error(f"Slack API says can't find channel {self.channel}")
+                    raise
+                case _:
+                    raise
 
     def describe_usergroup(
         self, handle: str
@@ -274,11 +287,13 @@ class SlackApi:
 
         channels_found = self.get_channels_by_names(self.channel)
         [channel_id] = [k for k in channels_found if channels_found[k] == self.channel]
+        slack_request.labels("conversations.info", "GET").inc()
+
         info = self._sc.conversations_info(channel=channel_id)
         if not info.data["channel"]["is_member"]:  # type: ignore[call-overload]
             self._sc.conversations_join(channel=channel_id)
 
-    def get_usergroup_id(self, handle: str) -> Optional[str]:
+    def get_usergroup_id(self, handle: str) -> str | None:
         try:
             return self.get_usergroup(handle)["id"]
         except UsergroupNotFoundException:
@@ -291,17 +306,26 @@ class SlackApi:
         :raises slack_sdk.errors.SlackApiError: if unsuccessful response from
         Slack API
         """
+        slack_request.labels("usergroups.list", "GET").inc()
+
         result = self._sc.usergroups_list(include_users=True)
         self.usergroups = result["usergroups"]
+        self._user_groups_initialized = True
 
     def get_usergroup(self, handle: str) -> dict[str, Any]:
+        if not self._user_groups_initialized:
+            self._initiate_usergroups()
         usergroup = [g for g in self.usergroups if g["handle"] == handle]
         if len(usergroup) != 1:
             raise UsergroupNotFoundException(handle)
         return usergroup[0]
 
     def create_usergroup(self, handle: str) -> str:
+        slack_request.labels("usergroups.create", "POST").inc()
+
         response = self._sc.usergroups_create(name=handle, handle=handle)
+        # Invalidate the usergroups list cache
+        self._user_groups_initialized = False
         return response["usergroup"]["id"]
 
     def update_usergroup(
@@ -317,6 +341,8 @@ class SlackApi:
         :raises slack_sdk.errors.SlackApiError: if unsuccessful response from
         Slack API
         """
+        slack_request.labels("usergroups.update", "POST").inc()
+
         self._sc.usergroups_update(
             usergroup=id, channels=channels_list, description=description
         )
@@ -337,6 +363,8 @@ class SlackApi:
             users_list = [self.get_random_deleted_user()]
 
         try:
+            slack_request.labels("usergroups.users.update", "POST").inc()
+
             self._sc.usergroups_users_update(usergroup=id, users=users_list)
         except SlackApiError as e:
             # Slack can throw an invalid_users error when emptying groups, but
@@ -365,10 +393,12 @@ class SlackApi:
         :raises UserNotFoundException: if the Slack user is not found
         """
         try:
+            slack_request.labels("users.lookupByEmail", "GET").inc()
+
             result = self._sc.users_lookupByEmail(email=f"{user_name}@{mail_address}")
         except SlackApiError as e:
             if e.response["error"] == "users_not_found":
-                raise UserNotFoundException(e.response["error"])
+                raise UserNotFoundException(e.response["error"]) from None
             raise
 
         return result["user"]["id"]
@@ -385,15 +415,23 @@ class SlackApi:
             k: v["name"] for k, v in self._get("channels").items() if k in channels_ids
         }
 
-    def get_users_by_names(self, user_names: Iterable[str]) -> dict[str, str]:
+    def get_active_users_by_names(self, user_names: Iterable[str]) -> dict[str, str]:
         return {
             k: v["name"]
             for k, v in self._get("users").items()
-            if v["name"] in user_names
+            if v["name"] in user_names and not v["deleted"]
         }
 
     def get_users_by_ids(self, users_ids: Iterable[str]) -> dict[str, str]:
-        return {k: v["name"] for k, v in self._get("users").items() if k in users_ids}
+        users = self._get("users")
+        translated_user_ids = (
+            self._translate_user_id(user_id) for user_id in users_ids
+        )
+        return {
+            user_id: user["name"]
+            for user_id in translated_user_ids
+            if (user := users.get(user_id))
+        }
 
     def _get(self, resource: str) -> dict[str, Any]:
         """
@@ -403,21 +441,23 @@ class SlackApi:
         :param resource: resource type
         :return: data from API call
         """
+        if resource in self._results:
+            return self._results[resource]
+
         result_key = "members" if resource == "users" else resource
         api_key = "conversations" if resource == "channels" else resource
         results = {}
-        additional_kwargs: dict[str, Union[str, int]] = {"cursor": ""}
-
-        if resource in self._results:
-            return self._results[resource]
+        additional_kwargs: dict[str, str | int] = {"cursor": ""}
 
         method_config = self.config.get_method_config(f"{api_key}.list")
         if method_config:
             additional_kwargs.update(method_config)
 
         while True:
+            slack_request.labels(f"{api_key}.list", "GET").inc()
+
             result = self._sc.api_call(
-                "{}.list".format(api_key), http_verb="GET", params=additional_kwargs
+                f"{api_key}.list", http_verb="GET", params=additional_kwargs
             )
 
             for r in result[result_key]:
@@ -425,10 +465,58 @@ class SlackApi:
 
             cursor = result["response_metadata"]["next_cursor"]
 
-            if cursor == "":
+            if not cursor:
                 break
 
             additional_kwargs["cursor"] = cursor
 
         self._results[resource] = results
+
+        if resource == "users":
+            self._enterprise_user_id_to_user_ids = {
+                enterprise_user_id: user["id"]
+                for user in results.values()
+                if (enterprise_user_id := user.get("enterprise_user", {}).get("id"))
+            }
         return results
+
+    def _translate_user_id(self, user_id: str) -> str:
+        """Translate enterprise user id to user id"""
+        return self._enterprise_user_id_to_user_ids.get(user_id, user_id)
+
+    def get_flat_conversation_history(
+        self, from_timestamp: int, to_timestamp: int | None
+    ) -> list[dict[str, Any]]:
+        """Calls conversation_history method to get all messages in a channel between
+        from_timestamp to to_timestamp ignoring threads"""
+        if not self.channel:
+            raise ValueError("Expecting self.channel to be set")
+
+        cursor = ""
+        responses = []
+        keep_fetching = True
+        while True:
+            slack_request.labels("conversations.history", "GET").inc()
+
+            response = self._sc.conversations_history(
+                cursor=cursor, channel=self.channel, **self.chat_kwargs
+            )
+
+            for r in response["messages"]:
+                if to_timestamp and float(r["ts"]) > to_timestamp:
+                    continue
+
+                if float(r["ts"]) < from_timestamp:
+                    keep_fetching = False
+                    break
+
+                responses.append(r)
+
+            if not keep_fetching:
+                break
+
+            cursor = response["response_metadata"]["next_cursor"]
+            if not cursor:
+                break
+
+        return responses

reconcile/utils/smtp_client.py

@@ -4,7 +4,6 @@ from email.header import Header
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 from email.utils import formataddr
-from typing import Optional
 
 from pydantic import (
     BaseModel,
@@ -54,7 +53,7 @@ class SmtpClient:
         self.passwd = server.password
         self.mail_address = mail_address
         self.timeout = timeout
-        self._client: Optional[smtplib.SMTP] = None
+        self._client: smtplib.SMTP | None = None
 
     @property
     def client(self) -> smtplib.SMTP:

reconcile/utils/sqs_gateway.py

@@ -13,7 +13,7 @@ class SQSGateway:
     """Wrapper around SQS AWS SDK"""
 
     def __init__(self, accounts, secret_reader: SecretReader):
-        queue_url = os.environ.get("gitlab_pr_submitter_queue_url")
+        queue_url = os.environ.get("gitlab_pr_submitter_queue_url")  # noqa: SIM112
         if not queue_url:
             raise SQSGatewayInitError(
                 "when /app-interface/app-interface-settings-1.yml#mergeRequestGateway "
@@ -46,17 +46,21 @@ class SQSGateway:
             a["name"] for a in accounts if a["uid"] == queue_account_uid
         ]
         if len(queue_account_name) != 1:
-            raise SQSGatewayInitError(
-                "account uid not found: {}".format(queue_account_uid)
-            )
+            raise SQSGatewayInitError(f"account uid not found: {queue_account_uid}")
         return queue_account_name[0]
 
     def send_message(self, body):
         self.sqs.send_message(QueueUrl=self.queue_url, MessageBody=json.dumps(body))
 
-    def receive_messages(self, visibility_timeout=30):
+    def receive_messages(
+        self,
+        visibility_timeout=30,
+        wait_time_seconds=20,
+    ):
         messages = self.sqs.receive_message(
-            QueueUrl=self.queue_url, VisibilityTimeout=visibility_timeout
+            QueueUrl=self.queue_url,
+            VisibilityTimeout=visibility_timeout,
+            WaitTimeSeconds=wait_time_seconds,
         ).get("Messages", [])
         return [(m["ReceiptHandle"], json.loads(m["Body"])) for m in messages]