mlrun 1.7.1rc10__py3-none-any.whl → 1.8.0rc11__py3-none-any.whl

mlrun/utils/notifications/notification/ipython.py

@@ -28,10 +28,11 @@ class IPythonNotification(NotificationBase):
 
     def __init__(
         self,
-        name: str = None,
-        params: dict[str, str] = None,
+        name: typing.Optional[str] = None,
+        params: typing.Optional[dict[str, str]] = None,
+        default_params: typing.Optional[dict[str, str]] = None,
     ):
-        super().__init__(name, params)
+        super().__init__(name, params, default_params)
         self._ipython = None
         try:
             import IPython
@@ -48,13 +49,13 @@ class IPythonNotification(NotificationBase):
     def push(
         self,
         message: str,
-        severity: typing.Union[
-            mlrun.common.schemas.NotificationSeverity, str
+        severity: typing.Optional[
+            typing.Union[mlrun.common.schemas.NotificationSeverity, str]
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
-        runs: typing.Union[mlrun.lists.RunList, list] = None,
-        custom_html: str = None,
-        alert: mlrun.common.schemas.AlertConfig = None,
-        event_data: mlrun.common.schemas.Event = None,
+        runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
+        custom_html: typing.Optional[typing.Optional[str]] = None,
+        alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
+        event_data: typing.Optional[mlrun.common.schemas.Event] = None,
     ):
         if not self._ipython:
             mlrun.utils.helpers.logger.debug(

mlrun/utils/notifications/notification/mail.py

@@ -0,0 +1,176 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+import typing
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+import aiosmtplib
+
+import mlrun.common.schemas
+import mlrun.lists
+import mlrun.utils.helpers
+import mlrun.utils.notifications.notification.base as base
+import mlrun.utils.regex
+
+DEFAULT_SMTP_PORT = 587
+
+
+class MailNotification(base.NotificationBase):
+    """
+    API/Client notification for sending run statuses as a mail message
+    """
+
+    boolean_params = ["use_tls", "start_tls", "validate_certs"]
+
+    required_params = [
+        "server_host",
+        "server_port",
+        "sender_address",
+        "username",
+        "password",
+        "email_addresses",
+    ] + boolean_params
+
+    @classmethod
+    def validate_params(cls, params):
+        for required_param in cls.required_params:
+            if required_param not in params:
+                raise ValueError(
+                    f"Parameter '{required_param}' is required for MailNotification"
+                )
+
+        for boolean_param in cls.boolean_params:
+            if not isinstance(params.get(boolean_param, None), bool):
+                raise ValueError(
+                    f"Parameter '{boolean_param}' must be a boolean for MailNotification"
+                )
+
+        cls._validate_emails(params)
+
+    async def push(
+        self,
+        message: str,
+        severity: typing.Optional[
+            typing.Union[mlrun.common.schemas.NotificationSeverity, str]
+        ] = mlrun.common.schemas.NotificationSeverity.INFO,
+        runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
+        custom_html: typing.Optional[typing.Optional[str]] = None,
+        alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
+        event_data: typing.Optional[mlrun.common.schemas.Event] = None,
+    ):
+        self.params["subject"] = f"[{severity}] {message}"
+        message_body_override = self.params.get("message_body_override", None)
+
+        runs_html = self._get_html(
+            message, severity, runs, custom_html, alert, event_data
+        )
+        self.params["body"] = runs_html
+
+        if message_body_override:
+            self.params["body"] = message_body_override.replace(
+                "{{ runs }}", runs_html
+            ).replace("{{runs}}", runs_html)
+
+        await self._send_email(**self.params)
+
+    @classmethod
+    def enrich_default_params(
+        cls, params: dict, default_params: typing.Optional[dict] = None
+    ) -> dict:
+        params = super().enrich_default_params(params, default_params)
+        params.setdefault("use_tls", True)
+        params.setdefault("start_tls", False)
+        params.setdefault("validate_certs", True)
+        params.setdefault("server_port", DEFAULT_SMTP_PORT)
+
+        default_mail_address = params.pop("default_email_addresses", "")
+        params["email_addresses"] = cls._merge_mail_addresses(
+            default_mail_address, params.get("email_addresses", "")
+        )
+
+        return params
+
+    @classmethod
+    def _merge_mail_addresses(
+        cls,
+        default_mail_address: typing.Union[str, list],
+        email_addresses: typing.Union[str, list],
+    ) -> str:
+        if isinstance(default_mail_address, str):
+            default_mail_address = (
+                default_mail_address.split(",") if default_mail_address else []
+            )
+        if isinstance(email_addresses, str):
+            email_addresses = email_addresses.split(",") if email_addresses else []
+        email_addresses.extend(default_mail_address)
+        email_addresses_str = ",".join(email_addresses)
+        return email_addresses_str
+
+    @classmethod
+    def _validate_emails(cls, params):
+        cls._validate_email_address(params["sender_address"])
+
+        if not isinstance(params["email_addresses"], (str, list)):
+            raise ValueError(
+                "Parameter 'email_addresses' must be a string or a list of strings"
+            )
+
+        email_addresses = params["email_addresses"]
+        if isinstance(email_addresses, str):
+            email_addresses = email_addresses.split(",")
+        for email_address in email_addresses:
+            cls._validate_email_address(email_address)
+
+    @classmethod
+    def _validate_email_address(cls, email_address):
+        if not isinstance(email_address, str):
+            raise ValueError(f"Email address '{email_address}' must be a string")
+
+        if not re.match(mlrun.utils.regex.mail_regex, email_address):
+            raise ValueError(f"Invalid email address '{email_address}'")
+
+    @staticmethod
+    async def _send_email(
+        email_addresses: str,
+        sender_address: str,
+        server_host: str,
+        server_port: int,
+        username: str,
+        password: str,
+        use_tls: bool,
+        start_tls: bool,
+        validate_certs: bool,
+        subject: str,
+        body: str,
+        **kwargs,
+    ):
+        # Create the email message
+        message = MIMEMultipart("alternative")
+        message["From"] = sender_address
+        message["To"] = email_addresses
+        message["Subject"] = subject
+        message.attach(MIMEText(body, "html"))
+
+        # Send the email
+        await aiosmtplib.send(
+            message,
+            hostname=server_host,
+            port=server_port,
+            username=username,
+            password=password,
+            use_tls=use_tls,
+            validate_certs=validate_certs,
+            start_tls=start_tls,
+        )

mlrun/utils/notifications/notification/slack.py

@@ -46,13 +46,13 @@ class SlackNotification(NotificationBase):
     async def push(
         self,
         message: str,
-        severity: typing.Union[
-            mlrun.common.schemas.NotificationSeverity, str
+        severity: typing.Optional[
+            typing.Union[mlrun.common.schemas.NotificationSeverity, str]
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
-        runs: typing.Union[mlrun.lists.RunList, list] = None,
-        custom_html: str = None,
-        alert: mlrun.common.schemas.AlertConfig = None,
-        event_data: mlrun.common.schemas.Event = None,
+        runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
+        custom_html: typing.Optional[typing.Optional[str]] = None,
+        alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
+        event_data: typing.Optional[mlrun.common.schemas.Event] = None,
     ):
         webhook = self.params.get("webhook", None) or mlrun.get_secret_or_env(
             "SLACK_WEBHOOK"

mlrun/utils/notifications/notification/webhook.py

@@ -37,13 +37,13 @@ class WebhookNotification(NotificationBase):
     async def push(
         self,
         message: str,
-        severity: typing.Union[
-            mlrun.common.schemas.NotificationSeverity, str
+        severity: typing.Optional[
+            typing.Union[mlrun.common.schemas.NotificationSeverity, str]
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
-        runs: typing.Union[mlrun.lists.RunList, list] = None,
-        custom_html: str = None,
-        alert: mlrun.common.schemas.AlertConfig = None,
-        event_data: mlrun.common.schemas.Event = None,
+        runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
+        custom_html: typing.Optional[typing.Optional[str]] = None,
+        alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
+        event_data: typing.Optional[mlrun.common.schemas.Event] = None,
     ):
         url = self.params.get("url", None)
         method = self.params.get("method", "post").lower()

mlrun/utils/notifications/notification_pusher.py

@@ -20,12 +20,8 @@ import traceback
 import typing
 from concurrent.futures import ThreadPoolExecutor
 
-import mlrun_pipelines.common.ops
-import mlrun_pipelines.models
-import mlrun_pipelines.utils
-
 import mlrun.common.constants as mlrun_constants
-import mlrun.common.runtimes.constants
+import mlrun.common.runtimes.constants as runtimes_constants
 import mlrun.common.schemas
 import mlrun.config
 import mlrun.db.base
@@ -33,11 +29,14 @@ import mlrun.errors
 import mlrun.lists
 import mlrun.model
 import mlrun.utils.helpers
+import mlrun.utils.notifications.notification as notification_module
+import mlrun.utils.notifications.notification.base as base
+import mlrun_pipelines.common.ops
+import mlrun_pipelines.models
+import mlrun_pipelines.utils
 from mlrun.utils import logger
 from mlrun.utils.condition_evaluator import evaluate_condition_in_separate_process
 
-from .notification import NotificationBase, NotificationTypes
-
 
 class _NotificationPusherBase:
     def _push(
@@ -100,31 +99,56 @@ class NotificationPusher(_NotificationPusherBase):
         "aborted": "{resource} aborted",
     }
 
-    def __init__(self, runs: typing.Union[mlrun.lists.RunList, list]):
+    def __init__(
+        self,
+        runs: typing.Union[mlrun.lists.RunList, list],
+        default_params: typing.Optional[dict] = None,
+    ):
         self._runs = runs
+        self._default_params = default_params or {}
         self._sync_notifications: list[
-            tuple[NotificationBase, mlrun.model.RunObject, mlrun.model.Notification]
+            tuple[
+                base.NotificationBase, mlrun.model.RunObject, mlrun.model.Notification
+            ]
         ] = []
         self._async_notifications: list[
-            tuple[NotificationBase, mlrun.model.RunObject, mlrun.model.Notification]
+            tuple[
+                base.NotificationBase, mlrun.model.RunObject, mlrun.model.Notification
+            ]
         ] = []
 
         for run in self._runs:
-            if isinstance(run, dict):
-                run = mlrun.model.RunObject.from_dict(run)
+            try:
+                self._process_run(run)
+            except Exception as exc:
+                logger.warning(
+                    "Failed to process run",
+                    run_uid=run.metadata.uid,
+                    error=mlrun.errors.err_to_str(exc),
+                )
 
-            for notification in run.spec.notifications:
-                try:
-                    notification.status = run.status.notifications.get(
-                        notification.name
-                    ).get("status", mlrun.common.schemas.NotificationStatus.PENDING)
-                except (AttributeError, KeyError):
-                    notification.status = (
-                        mlrun.common.schemas.NotificationStatus.PENDING
-                    )
+    def _process_run(self, run):
+        if isinstance(run, dict):
+            run = mlrun.model.RunObject.from_dict(run)
+
+        for notification in run.spec.notifications:
+            try:
+                self._process_notification(notification, run)
+            except Exception as exc:
+                logger.warning(
+                    "Failed to process notification",
+                    run_uid=run.metadata.uid,
+                    notification=notification,
+                    error=mlrun.errors.err_to_str(exc),
+                )
 
-                if self._should_notify(run, notification):
-                    self._load_notification(run, notification)
+    def _process_notification(self, notification, run):
+        notification.status = run.status.notifications.get(notification.name, {}).get(
+            "status",
+            mlrun.common.schemas.NotificationStatus.PENDING,
+        )
+        if self._should_notify(run, notification):
+            self._load_notification(run, notification)
 
     def push(self):
         """
@@ -168,6 +192,11 @@ class NotificationPusher(_NotificationPusherBase):
                     logger.warning(
                         "Failed to push notification async",
                         error=mlrun.errors.err_to_str(result),
+                        traceback=traceback.format_exception(
+                            etype=type(result),
+                            value=result,
+                            tb=result.__traceback__,
+                        ),
                     )
 
         logger.debug(
@@ -197,7 +226,7 @@ class NotificationPusher(_NotificationPusherBase):
         for when_state in when_states:
             if when_state == run_state:
                 if (
-                    run_state == "completed"
+                    run_state == runtimes_constants.RunStates.completed
                     and evaluate_condition_in_separate_process(
                         notification.condition,
                         context={
@@ -205,22 +234,29 @@ class NotificationPusher(_NotificationPusherBase):
                             "notification": notification.to_dict(),
                         },
                     )
-                ) or run_state in ["error", "aborted"]:
+                ) or run_state in [
+                    runtimes_constants.RunStates.error,
+                    runtimes_constants.RunStates.aborted,
+                    runtimes_constants.RunStates.running,
+                ]:
                     return True
 
         return False
 
     def _load_notification(
         self, run: mlrun.model.RunObject, notification_object: mlrun.model.Notification
-    ) -> NotificationBase:
+    ) -> base.NotificationBase:
         name = notification_object.name
-        notification_type = NotificationTypes(
-            notification_object.kind or NotificationTypes.console
+        notification_type = notification_module.NotificationTypes(
+            notification_object.kind or notification_module.NotificationTypes.console
         )
         params = {}
         params.update(notification_object.secret_params)
         params.update(notification_object.params)
-        notification = notification_type.get_notification()(name, params)
+        default_params = self._default_params.get(notification_type.value, {})
+        notification = notification_type.get_notification()(
+            name, params, default_params
+        )
         if notification.is_async:
             self._async_notifications.append((notification, run, notification_object))
         else:
@@ -260,7 +296,7 @@ class NotificationPusher(_NotificationPusherBase):
 
     def _push_notification_sync(
         self,
-        notification: NotificationBase,
+        notification: base.NotificationBase,
         run: mlrun.model.RunObject,
         notification_object: mlrun.model.Notification,
     ):
@@ -308,7 +344,7 @@ class NotificationPusher(_NotificationPusherBase):
 
     async def _push_notification_async(
         self,
-        notification: NotificationBase,
+        notification: base.NotificationBase,
         run: mlrun.model.RunObject,
         notification_object: mlrun.model.Notification,
     ):
@@ -361,7 +397,7 @@ class NotificationPusher(_NotificationPusherBase):
         run_uid: str,
         project: str,
         notification: mlrun.model.Notification,
-        status: str = None,
+        status: typing.Optional[str] = None,
         sent_time: typing.Optional[datetime.datetime] = None,
         reason: typing.Optional[str] = None,
     ):
@@ -409,7 +445,7 @@ class NotificationPusher(_NotificationPusherBase):
             _run["step_kind"] = _step.step_type
             if _step.skipped:
                 _run.setdefault("status", {})["state"] = (
-                    mlrun.common.runtimes.constants.RunStates.skipped
+                    runtimes_constants.RunStates.skipped
                 )
             steps.append(_run)
 
@@ -436,7 +472,7 @@ class NotificationPusher(_NotificationPusherBase):
                 if _step.skipped:
                     state = mlrun.common.schemas.FunctionState.skipped
                 else:
-                    state = mlrun.common.runtimes.constants.PodPhases.pod_phase_to_run_state(
+                    state = runtimes_constants.PodPhases.pod_phase_to_run_state(
                         pod_phase
                     )
                 function["status"] = {"state": state}
@@ -516,9 +552,11 @@ class NotificationPusher(_NotificationPusherBase):
 
 
 class CustomNotificationPusher(_NotificationPusherBase):
-    def __init__(self, notification_types: list[str] = None):
+    def __init__(self, notification_types: typing.Optional[list[str]] = None):
         notifications = {
-            notification_type: NotificationTypes(notification_type).get_notification()()
+            notification_type: notification_module.NotificationTypes(
+                notification_type
+            ).get_notification()()
             for notification_type in notification_types
         }
         self._sync_notifications = {
@@ -545,7 +583,7 @@ class CustomNotificationPusher(_NotificationPusherBase):
             mlrun.common.schemas.NotificationSeverity, str
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
-        custom_html: str = None,
+        custom_html: typing.Optional[str] = None,
     ):
         def sync_push():
             for notification_type, notification in self._sync_notifications.items():
@@ -567,14 +605,16 @@ class CustomNotificationPusher(_NotificationPusherBase):
     def add_notification(
         self,
         notification_type: str,
-        params: dict[str, str] = None,
+        params: typing.Optional[dict[str, str]] = None,
     ):
         if notification_type in self._async_notifications:
             self._async_notifications[notification_type].load_notification(params)
         elif notification_type in self._sync_notifications:
             self._sync_notifications[notification_type].load_notification(params)
         else:
-            notification = NotificationTypes(notification_type).get_notification()(
+            notification = notification_module.NotificationTypes(
+                notification_type
+            ).get_notification()(
                 params=params,
             )
             if notification.is_async:
@@ -592,7 +632,9 @@ class CustomNotificationPusher(_NotificationPusherBase):
         else:
             logger.warning(f"No notification of type {notification_type} in project")
 
-    def edit_notification(self, notification_type: str, params: dict[str, str] = None):
+    def edit_notification(
+        self, notification_type: str, params: typing.Optional[dict[str, str]] = None
+    ):
         self.remove_notification(notification_type)
         self.add_notification(notification_type, params)
 
@@ -606,7 +648,7 @@ class CustomNotificationPusher(_NotificationPusherBase):
 
         # get notification's inverse dependencies, and only push the notification if
         # none of its inverse dependencies are being sent
-        inverse_dependencies = NotificationTypes(
+        inverse_dependencies = notification_module.NotificationTypes(
             notification_type
         ).inverse_dependencies()
         for inverse_dependency in inverse_dependencies:
@@ -622,8 +664,8 @@ class CustomNotificationPusher(_NotificationPusherBase):
     def push_pipeline_start_message(
         self,
         project: str,
-        commit_id: str = None,
-        pipeline_id: str = None,
+        commit_id: typing.Optional[str] = None,
+        pipeline_id: typing.Optional[str] = None,
         has_workflow_url: bool = False,
     ):
         message = f"Workflow started in project {project}"
@@ -651,7 +693,7 @@ class CustomNotificationPusher(_NotificationPusherBase):
         self,
         runs: typing.Union[mlrun.lists.RunList, list],
         push_all: bool = False,
-        state: str = None,
+        state: typing.Optional[str] = None,
     ):
         """
         push a structured table with run results to notification targets

mlrun/utils/regex.py

@@ -86,7 +86,7 @@ tag_name = label_value
 
 secret_key = k8s_secret_and_config_map_key
 
-artifact_key = [r"[^\/\\]+$"]
+artifact_key = [r"^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$"]
 
 # must not start with _
 # must be alphanumeric or _
@@ -96,8 +96,17 @@ v3io_stream_consumer_group = [r"^(?!_)[a-zA-Z0-9_]{1,256}$"]
 # URI patterns
 run_uri_pattern = r"^(?P<project>.*)@(?P<uid>.*)\#(?P<iteration>.*?)(:(?P<tag>.*))?$"
 
-artifact_uri_pattern = r"^((?P<project>.*)/)?(?P<key>.*?)(\#(?P<iteration>.*?))?(:(?P<tag>.*?))?(@(?P<tree>.*))?$"
+artifact_uri_pattern = (
+    r"^((?P<project>.*)/)?"  # Optional project
+    r"(?P<key>.*?)"  # Key
+    r"(\#(?P<iteration>.*?))?"  # Optional iteration
+    r"(:(?P<tag>.*?))?"  # Optional tag
+    r"(@(?P<tree>.*?))?"  # Optional tree
+    r"(\^(?P<uid>.*))?$"  # Optional uid
+)
 
 artifact_producer_uri_pattern = (
     r"^((?P<project>.*)/)?(?P<uid>.*?)(\-(?P<iteration>.*?))?$"
 )
+
+mail_regex = r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$"

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "837dd64ab75493fe34986c6203d884a0ae33843b",
-  "version": "1.7.1-rc10"
+  "git_commit": "0b5788c2600a7a91fa5b16a68b8bd5a65937f1f5",
+  "version": "1.8.0-rc11"
 }