mlrun 1.7.2rc3__py3-none-any.whl → 1.8.0rc2__py3-none-any.whl

mlrun/utils/notifications/notification/mail.py

@@ -0,0 +1,149 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+import typing
+from email.message import EmailMessage
+
+import aiosmtplib
+
+import mlrun.common.schemas
+import mlrun.lists
+import mlrun.utils.helpers
+import mlrun.utils.notifications.notification.base as base
+import mlrun.utils.regex
+
+DEFAULT_SMTP_PORT = 587
+
+
+class MailNotification(base.NotificationBase):
+    """
+    API/Client notification for sending run statuses as a mail message
+    """
+
+    boolean_params = ["use_tls", "start_tls", "validate_certs"]
+
+    required_params = [
+        "server_host",
+        "server_port",
+        "sender_address",
+        "username",
+        "password",
+        "email_addresses",
+    ] + boolean_params
+
+    @classmethod
+    def validate_params(cls, params):
+        for required_param in cls.required_params:
+            if required_param not in params:
+                raise ValueError(
+                    f"Parameter '{required_param}' is required for MailNotification"
+                )
+
+        for boolean_param in cls.boolean_params:
+            if not isinstance(params.get(boolean_param, None), bool):
+                raise ValueError(
+                    f"Parameter '{boolean_param}' must be a boolean for MailNotification"
+                )
+
+        cls._validate_emails(params)
+
+    async def push(
+        self,
+        message: str,
+        severity: typing.Optional[
+            typing.Union[mlrun.common.schemas.NotificationSeverity, str]
+        ] = mlrun.common.schemas.NotificationSeverity.INFO,
+        runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
+        custom_html: typing.Optional[typing.Optional[str]] = None,
+        alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
+        event_data: typing.Optional[mlrun.common.schemas.Event] = None,
+    ):
+        self.params.setdefault("subject", f"[{severity}] {message}")
+        self.params.setdefault("body", message)
+        await self._send_email(**self.params)
+
+    @classmethod
+    def enrich_default_params(
+        cls, params: dict, default_params: typing.Optional[dict] = None
+    ) -> dict:
+        params = super().enrich_default_params(params, default_params)
+        params.setdefault("use_tls", True)
+        params.setdefault("start_tls", False)
+        params.setdefault("validate_certs", True)
+        params.setdefault("server_port", DEFAULT_SMTP_PORT)
+
+        default_mail_address = params.pop("default_email_addresses", "")
+        email_addresses = params.get("email_addresses", default_mail_address)
+        if isinstance(email_addresses, list):
+            email_addresses = ",".join(email_addresses)
+        params["email_addresses"] = email_addresses
+
+        return params
+
+    @classmethod
+    def _validate_emails(cls, params):
+        cls._validate_email_address(params["sender_address"])
+
+        if not isinstance(params["email_addresses"], (str, list)):
+            raise ValueError(
+                "Parameter 'email_addresses' must be a string or a list of strings"
+            )
+
+        email_addresses = params["email_addresses"]
+        if isinstance(email_addresses, str):
+            email_addresses = email_addresses.split(",")
+        for email_address in email_addresses:
+            cls._validate_email_address(email_address)
+
+    @classmethod
+    def _validate_email_address(cls, email_address):
+        if not isinstance(email_address, str):
+            raise ValueError(f"Email address '{email_address}' must be a string")
+
+        if not re.match(mlrun.utils.regex.mail_regex, email_address):
+            raise ValueError(f"Invalid email address '{email_address}'")
+
+    @staticmethod
+    async def _send_email(
+        email_addresses: str,
+        sender_address: str,
+        server_host: str,
+        server_port: int,
+        username: str,
+        password: str,
+        use_tls: bool,
+        start_tls: bool,
+        validate_certs: bool,
+        subject: str,
+        body: str,
+        **kwargs,
+    ):
+        # Create the email message
+        message = EmailMessage()
+        message["From"] = sender_address
+        message["To"] = email_addresses
+        message["Subject"] = subject
+        message.set_content(body)
+
+        # Send the email
+        await aiosmtplib.send(
+            message,
+            hostname=server_host,
+            port=server_port,
+            username=username,
+            password=password,
+            use_tls=use_tls,
+            validate_certs=validate_certs,
+            start_tls=start_tls,
+        )

mlrun/utils/notifications/notification/slack.py

@@ -46,13 +46,13 @@ class SlackNotification(NotificationBase):
     async def push(
         self,
         message: str,
-        severity: typing.Union[
-            mlrun.common.schemas.NotificationSeverity, str
+        severity: typing.Optional[
+            typing.Union[mlrun.common.schemas.NotificationSeverity, str]
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
-        runs: typing.Union[mlrun.lists.RunList, list] = None,
-        custom_html: str = None,
-        alert: mlrun.common.schemas.AlertConfig = None,
-        event_data: mlrun.common.schemas.Event = None,
+        runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
+        custom_html: typing.Optional[typing.Optional[str]] = None,
+        alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
+        event_data: typing.Optional[mlrun.common.schemas.Event] = None,
     ):
         webhook = self.params.get("webhook", None) or mlrun.get_secret_or_env(
             "SLACK_WEBHOOK"

mlrun/utils/notifications/notification/webhook.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import re
 import typing
 
 import aiohttp
@@ -38,13 +37,13 @@ class WebhookNotification(NotificationBase):
     async def push(
         self,
         message: str,
-        severity: typing.Union[
-            mlrun.common.schemas.NotificationSeverity, str
+        severity: typing.Optional[
+            typing.Union[mlrun.common.schemas.NotificationSeverity, str]
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
-        runs: typing.Union[mlrun.lists.RunList, list] = None,
-        custom_html: str = None,
-        alert: mlrun.common.schemas.AlertConfig = None,
-        event_data: mlrun.common.schemas.Event = None,
+        runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
+        custom_html: typing.Optional[typing.Optional[str]] = None,
+        alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
+        event_data: typing.Optional[mlrun.common.schemas.Event] = None,
     ):
         url = self.params.get("url", None)
         method = self.params.get("method", "post").lower()
@@ -94,6 +93,7 @@ class WebhookNotification(NotificationBase):
 
     @staticmethod
     def _serialize_runs_in_request_body(override_body, runs):
+        str_parsed_runs = ""
         runs = runs or []
 
         def parse_runs():
@@ -105,26 +105,22 @@ class WebhookNotification(NotificationBase):
                     parsed_run = {
                         "project": run["metadata"]["project"],
                         "name": run["metadata"]["name"],
+                        "host": run["metadata"]["labels"]["host"],
                         "status": {"state": run["status"]["state"]},
                     }
-                    if host := run["metadata"].get("labels", {}).get("host", ""):
-                        parsed_run["host"] = host
-                    if error := run["status"].get("error"):
-                        parsed_run["status"]["error"] = error
-                    elif results := run["status"].get("results"):
-                        parsed_run["status"]["results"] = results
+                    if run["status"].get("error", None):
+                        parsed_run["status"]["error"] = run["status"]["error"]
+                    elif run["status"].get("results", None):
+                        parsed_run["status"]["results"] = run["status"]["results"]
                     parsed_runs.append(parsed_run)
             return str(parsed_runs)
 
         if isinstance(override_body, dict):
             for key, value in override_body.items():
-                if not isinstance(value, str):
-                    # If the value is not a string, we don't want to parse it
-                    continue
-                if re.search(r"{{\s*runs\s*}}", value):
-                    str_parsed_runs = parse_runs()
-                    override_body[key] = re.sub(
-                        r"{{\s*runs\s*}}", str_parsed_runs, value
-                    )
-
+                if "{{ runs }}" or "{{runs}}" in value:
+                    if not str_parsed_runs:
+                        str_parsed_runs = parse_runs()
+                    override_body[key] = value.replace(
+                        "{{ runs }}", str_parsed_runs
+                    ).replace("{{runs}}", str_parsed_runs)
         return override_body

mlrun/utils/notifications/notification_pusher.py

@@ -20,10 +20,6 @@ import traceback
 import typing
 from concurrent.futures import ThreadPoolExecutor
 
-import mlrun_pipelines.common.ops
-import mlrun_pipelines.models
-import mlrun_pipelines.utils
-
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.runtimes.constants
 import mlrun.common.schemas
@@ -33,11 +29,14 @@ import mlrun.errors
 import mlrun.lists
 import mlrun.model
 import mlrun.utils.helpers
+import mlrun.utils.notifications.notification as notification_module
+import mlrun.utils.notifications.notification.base as base
+import mlrun_pipelines.common.ops
+import mlrun_pipelines.models
+import mlrun_pipelines.utils
 from mlrun.utils import logger
 from mlrun.utils.condition_evaluator import evaluate_condition_in_separate_process
 
-from .notification import NotificationBase, NotificationTypes
-
 
 class _NotificationPusherBase:
     def _push(
@@ -100,13 +99,22 @@ class NotificationPusher(_NotificationPusherBase):
         "aborted": "{resource} aborted",
     }
 
-    def __init__(self, runs: typing.Union[mlrun.lists.RunList, list]):
+    def __init__(
+        self,
+        runs: typing.Union[mlrun.lists.RunList, list],
+        default_params: typing.Optional[dict] = None,
+    ):
         self._runs = runs
+        self._default_params = default_params or {}
         self._sync_notifications: list[
-            tuple[NotificationBase, mlrun.model.RunObject, mlrun.model.Notification]
+            tuple[
+                base.NotificationBase, mlrun.model.RunObject, mlrun.model.Notification
+            ]
         ] = []
         self._async_notifications: list[
-            tuple[NotificationBase, mlrun.model.RunObject, mlrun.model.Notification]
+            tuple[
+                base.NotificationBase, mlrun.model.RunObject, mlrun.model.Notification
+            ]
         ] = []
 
         for run in self._runs:
@@ -168,11 +176,6 @@ class NotificationPusher(_NotificationPusherBase):
                     logger.warning(
                         "Failed to push notification async",
                         error=mlrun.errors.err_to_str(result),
-                        traceback=traceback.format_exception(
-                            etype=type(result),
-                            value=result,
-                            tb=result.__traceback__,
-                        ),
                     )
 
         logger.debug(
@@ -217,15 +220,18 @@ class NotificationPusher(_NotificationPusherBase):
 
     def _load_notification(
         self, run: mlrun.model.RunObject, notification_object: mlrun.model.Notification
-    ) -> NotificationBase:
+    ) -> base.NotificationBase:
         name = notification_object.name
-        notification_type = NotificationTypes(
-            notification_object.kind or NotificationTypes.console
+        notification_type = notification_module.NotificationTypes(
+            notification_object.kind or notification_module.NotificationTypes.console
         )
         params = {}
         params.update(notification_object.secret_params)
         params.update(notification_object.params)
-        notification = notification_type.get_notification()(name, params)
+        default_params = self._default_params.get(notification_type.value, {})
+        notification = notification_type.get_notification()(
+            name, params, default_params
+        )
         if notification.is_async:
             self._async_notifications.append((notification, run, notification_object))
         else:
@@ -265,7 +271,7 @@ class NotificationPusher(_NotificationPusherBase):
 
     def _push_notification_sync(
         self,
-        notification: NotificationBase,
+        notification: base.NotificationBase,
         run: mlrun.model.RunObject,
         notification_object: mlrun.model.Notification,
     ):
@@ -313,7 +319,7 @@ class NotificationPusher(_NotificationPusherBase):
 
     async def _push_notification_async(
         self,
-        notification: NotificationBase,
+        notification: base.NotificationBase,
         run: mlrun.model.RunObject,
         notification_object: mlrun.model.Notification,
     ):
@@ -366,7 +372,7 @@ class NotificationPusher(_NotificationPusherBase):
         run_uid: str,
         project: str,
         notification: mlrun.model.Notification,
-        status: str = None,
+        status: typing.Optional[str] = None,
         sent_time: typing.Optional[datetime.datetime] = None,
         reason: typing.Optional[str] = None,
     ):
@@ -521,9 +527,11 @@ class NotificationPusher(_NotificationPusherBase):
 
 
 class CustomNotificationPusher(_NotificationPusherBase):
-    def __init__(self, notification_types: list[str] = None):
+    def __init__(self, notification_types: typing.Optional[list[str]] = None):
         notifications = {
-            notification_type: NotificationTypes(notification_type).get_notification()()
+            notification_type: notification_module.NotificationTypes(
+                notification_type
+            ).get_notification()()
             for notification_type in notification_types
         }
         self._sync_notifications = {
@@ -550,7 +558,7 @@ class CustomNotificationPusher(_NotificationPusherBase):
             mlrun.common.schemas.NotificationSeverity, str
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
-        custom_html: str = None,
+        custom_html: typing.Optional[str] = None,
     ):
         def sync_push():
             for notification_type, notification in self._sync_notifications.items():
@@ -572,14 +580,16 @@ class CustomNotificationPusher(_NotificationPusherBase):
     def add_notification(
         self,
         notification_type: str,
-        params: dict[str, str] = None,
+        params: typing.Optional[dict[str, str]] = None,
     ):
         if notification_type in self._async_notifications:
             self._async_notifications[notification_type].load_notification(params)
         elif notification_type in self._sync_notifications:
             self._sync_notifications[notification_type].load_notification(params)
         else:
-            notification = NotificationTypes(notification_type).get_notification()(
+            notification = notification_module.NotificationTypes(
+                notification_type
+            ).get_notification()(
                 params=params,
             )
             if notification.is_async:
@@ -597,7 +607,9 @@ class CustomNotificationPusher(_NotificationPusherBase):
         else:
             logger.warning(f"No notification of type {notification_type} in project")
 
-    def edit_notification(self, notification_type: str, params: dict[str, str] = None):
+    def edit_notification(
+        self, notification_type: str, params: typing.Optional[dict[str, str]] = None
+    ):
         self.remove_notification(notification_type)
         self.add_notification(notification_type, params)
 
@@ -611,7 +623,7 @@ class CustomNotificationPusher(_NotificationPusherBase):
 
         # get notification's inverse dependencies, and only push the notification if
         # none of its inverse dependencies are being sent
-        inverse_dependencies = NotificationTypes(
+        inverse_dependencies = notification_module.NotificationTypes(
             notification_type
         ).inverse_dependencies()
         for inverse_dependency in inverse_dependencies:
@@ -627,8 +639,8 @@ class CustomNotificationPusher(_NotificationPusherBase):
     def push_pipeline_start_message(
         self,
         project: str,
-        commit_id: str = None,
-        pipeline_id: str = None,
+        commit_id: typing.Optional[str] = None,
+        pipeline_id: typing.Optional[str] = None,
         has_workflow_url: bool = False,
     ):
         message = f"Workflow started in project {project}"
@@ -656,7 +668,7 @@ class CustomNotificationPusher(_NotificationPusherBase):
         self,
         runs: typing.Union[mlrun.lists.RunList, list],
         push_all: bool = False,
-        state: str = None,
+        state: typing.Optional[str] = None,
     ):
         """
         push a structured table with run results to notification targets

mlrun/utils/regex.py

@@ -86,7 +86,7 @@ tag_name = label_value
 
 secret_key = k8s_secret_and_config_map_key
 
-artifact_key = [r"[^\/\\]+$"]
+artifact_key = [r"^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$"]
 
 # must not start with _
 # must be alphanumeric or _
@@ -101,3 +101,5 @@ artifact_uri_pattern = r"^((?P<project>.*)/)?(?P<key>.*?)(\#(?P<iteration>.*?))?
 artifact_producer_uri_pattern = (
     r"^((?P<project>.*)/)?(?P<uid>.*?)(\-(?P<iteration>.*?))?$"
 )
+
+mail_regex = r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$"

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "3440c50863afd3b3dd850ade4ee237e8a1eb3211",
-  "version": "1.7.2-rc3"
+  "git_commit": "d1bac86097b49ad1b459421723c455622e88e5b3",
+  "version": "1.8.0-rc2"
 }

{mlrun-1.7.2rc3.dist-info → mlrun-1.8.0rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.7.2rc3
+Version: 1.8.0rc2
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -23,35 +23,37 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: urllib3<1.27,>=1.26.9
 Requires-Dist: GitPython>=3.1.41,~=3.1
-Requires-Dist: aiohttp<3.11,>=3.9
+Requires-Dist: aiohttp~=3.10.0
 Requires-Dist: aiohttp-retry~=2.8.0
 Requires-Dist: click~=8.1
 Requires-Dist: nest-asyncio~=1.0
 Requires-Dist: ipython~=8.10
-Requires-Dist: nuclio-jupyter~=0.10.4
-Requires-Dist: numpy<1.27.0,>=1.16.5
+Requires-Dist: nuclio-jupyter~=0.11.1
+Requires-Dist: numpy<1.27.0,>=1.26.4
 Requires-Dist: pandas<2.2,>=1.2
 Requires-Dist: pyarrow<18,>=10.0
 Requires-Dist: pyyaml<7,>=5.4.1
 Requires-Dist: requests~=2.32
 Requires-Dist: tabulate~=0.8.6
 Requires-Dist: v3io~=0.6.9
-Requires-Dist: pydantic<1.10.15,>=1.10.8
+Requires-Dist: pydantic>=1.10.15
 Requires-Dist: mergedeep~=1.3
 Requires-Dist: v3io-frames~=0.10.14
 Requires-Dist: semver~=3.0
 Requires-Dist: dependency-injector~=4.41
 Requires-Dist: fsspec<2024.7,>=2023.9.2
 Requires-Dist: v3iofs~=0.1.17
-Requires-Dist: storey~=1.7.50
+Requires-Dist: storey~=1.8.0
 Requires-Dist: inflection~=0.5.0
-Requires-Dist: python-dotenv~=0.17.0
-Requires-Dist: setuptools~=71.0
+Requires-Dist: python-dotenv~=1.0
+Requires-Dist: setuptools>=75.2
 Requires-Dist: deprecated~=1.2
 Requires-Dist: jinja2>=3.1.3,~=3.1
 Requires-Dist: orjson<4,>=3.9.15
-Requires-Dist: mlrun-pipelines-kfp-common~=0.1.9
-Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.1.6
+Requires-Dist: mlrun-pipelines-kfp-common~=0.2.2
+Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.2.1
+Requires-Dist: docstring_parser~=0.16
+Requires-Dist: aiosmtplib~=3.0
 Provides-Extra: s3
 Requires-Dist: boto3<1.36,>=1.28.0; extra == "s3"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "s3"
@@ -102,9 +104,9 @@ Requires-Dist: snowflake-connector-python~=3.7; extra == "snowflake"
 Provides-Extra: api
 Requires-Dist: uvicorn~=0.27.1; extra == "api"
 Requires-Dist: dask-kubernetes~=0.11.0; extra == "api"
-Requires-Dist: apscheduler==3.10.3; extra == "api"
+Requires-Dist: apscheduler<4,>=3.10.3; extra == "api"
 Requires-Dist: objgraph~=3.6; extra == "api"
-Requires-Dist: igz-mgmt~=0.2.0; extra == "api"
+Requires-Dist: igz-mgmt~=0.4.1; extra == "api"
 Requires-Dist: humanfriendly~=10.0; extra == "api"
 Requires-Dist: fastapi~=0.110.0; extra == "api"
 Requires-Dist: sqlalchemy~=1.4; extra == "api"
@@ -112,6 +114,7 @@ Requires-Dist: pymysql~=1.0; extra == "api"
 Requires-Dist: alembic~=1.9; extra == "api"
 Requires-Dist: timelength~=1.1; extra == "api"
 Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "api"
+Requires-Dist: aiosmtplib~=3.0; extra == "api"
 Provides-Extra: all
 Requires-Dist: adlfs==2023.9.0; extra == "all"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "all"
@@ -176,8 +179,9 @@ Requires-Dist: taoswswrap~=0.2.0; extra == "complete"
 Provides-Extra: complete-api
 Requires-Dist: adlfs==2023.9.0; extra == "complete-api"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete-api"
+Requires-Dist: aiosmtplib~=3.0; extra == "complete-api"
 Requires-Dist: alembic~=1.9; extra == "complete-api"
-Requires-Dist: apscheduler==3.10.3; extra == "complete-api"
+Requires-Dist: apscheduler<4,>=3.10.3; extra == "complete-api"
 Requires-Dist: avro~=1.11; extra == "complete-api"
 Requires-Dist: azure-core~=1.24; extra == "complete-api"
 Requires-Dist: azure-identity~=1.5; extra == "complete-api"
@@ -195,7 +199,7 @@ Requires-Dist: google-cloud-storage==2.14.0; extra == "complete-api"
 Requires-Dist: google-cloud==0.34; extra == "complete-api"
 Requires-Dist: graphviz~=0.20.0; extra == "complete-api"
 Requires-Dist: humanfriendly~=10.0; extra == "complete-api"
-Requires-Dist: igz-mgmt~=0.2.0; extra == "complete-api"
+Requires-Dist: igz-mgmt~=0.4.1; extra == "complete-api"
 Requires-Dist: kafka-python~=2.0; extra == "complete-api"
 Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "complete-api"
 Requires-Dist: mlflow~=2.8; extra == "complete-api"