mlrun 1.8.0rc1__py3-none-any.whl → 1.8.0rc3__py3-none-any.whl

mlrun/runtimes/pod.py

@@ -17,21 +17,22 @@ import os
 import re
 import time
 import typing
+from collections.abc import Iterable
 from enum import Enum
 
 import dotenv
 import kubernetes.client as k8s_client
+from kubernetes.client import V1Volume, V1VolumeMount
 
 import mlrun.common.constants
 import mlrun.errors
+import mlrun.runtimes.mounts
 import mlrun.utils.regex
-import mlrun_pipelines.mounts
 from mlrun.common.schemas import (
     NodeSelectorOperator,
     PreemptionModes,
     SecurityContextEnrichmentModes,
 )
-from mlrun_pipelines.mixins import KfpAdapterMixin
 
 from ..config import config as mlconf
 from ..k8s_utils import (
@@ -367,6 +368,35 @@ class KubeResourceSpec(FunctionSpec):
                 + f"service accounts {allowed_service_accounts}"
             )
 
+    def with_volumes(
+        self,
+        volumes: typing.Union[list[dict], dict, V1Volume],
+    ) -> "KubeResourceSpec":
+        """Add volumes to the volumes dictionary, only used as part of the mlrun_pipelines mount functions."""
+        if isinstance(volumes, dict):
+            set_named_item(self._volumes, volumes)
+        elif isinstance(volumes, Iterable):
+            for volume in volumes:
+                set_named_item(self._volumes, volume)
+        else:
+            set_named_item(self._volumes, volumes)
+        return self
+
+    def with_volume_mounts(
+        self,
+        volume_mounts: typing.Union[list[dict], dict, V1VolumeMount],
+    ) -> "KubeResourceSpec":
+        """Add volume mounts to the volume mounts dictionary,
+        only used as part of the mlrun_pipelines mount functions."""
+        if isinstance(volume_mounts, dict):
+            self._set_volume_mount(volume_mounts)
+        elif isinstance(volume_mounts, Iterable):
+            for volume_mount in volume_mounts:
+                self._set_volume_mount(volume_mount)
+        else:
+            self._set_volume_mount(volume_mounts)
+        return self
+
     def _set_volume_mount(
         self, volume_mount, volume_mounts_field_name="_volume_mounts"
     ):
@@ -942,12 +972,12 @@ class AutoMountType(str, Enum):
     @classmethod
     def all_mount_modifiers(cls):
         return [
-            mlrun_pipelines.mounts.v3io_cred.__name__,
-            mlrun_pipelines.mounts.mount_v3io.__name__,
-            mlrun_pipelines.mounts.mount_pvc.__name__,
-            mlrun_pipelines.mounts.auto_mount.__name__,
-            mlrun_pipelines.mounts.mount_s3.__name__,
-            mlrun_pipelines.mounts.set_env_variables.__name__,
+            mlrun.runtimes.mounts.v3io_cred.__name__,
+            mlrun.runtimes.mounts.mount_v3io.__name__,
+            mlrun.runtimes.mounts.mount_pvc.__name__,
+            mlrun.runtimes.mounts.auto_mount.__name__,
+            mlrun.runtimes.mounts.mount_s3.__name__,
+            mlrun.runtimes.mounts.set_env_variables.__name__,
         ]
 
     @classmethod
@@ -964,27 +994,27 @@ class AutoMountType(str, Enum):
     def _get_auto_modifier():
         # If we're running on Iguazio - use v3io_cred
         if mlconf.igz_version != "":
-            return mlrun_pipelines.mounts.v3io_cred
+            return mlrun.runtimes.mounts.v3io_cred
         # Else, either pvc mount if it's configured or do nothing otherwise
         pvc_configured = (
             "MLRUN_PVC_MOUNT" in os.environ
             or "pvc_name" in mlconf.get_storage_auto_mount_params()
         )
-        return mlrun_pipelines.mounts.mount_pvc if pvc_configured else None
+        return mlrun.runtimes.mounts.mount_pvc if pvc_configured else None
 
     def get_modifier(self):
         return {
             AutoMountType.none: None,
-            AutoMountType.v3io_credentials: mlrun_pipelines.mounts.v3io_cred,
-            AutoMountType.v3io_fuse: mlrun_pipelines.mounts.mount_v3io,
-            AutoMountType.pvc: mlrun_pipelines.mounts.mount_pvc,
+            AutoMountType.v3io_credentials: mlrun.runtimes.mounts.v3io_cred,
+            AutoMountType.v3io_fuse: mlrun.runtimes.mounts.mount_v3io,
+            AutoMountType.pvc: mlrun.runtimes.mounts.mount_pvc,
             AutoMountType.auto: self._get_auto_modifier(),
-            AutoMountType.s3: mlrun_pipelines.mounts.mount_s3,
-            AutoMountType.env: mlrun_pipelines.mounts.set_env_variables,
+            AutoMountType.s3: mlrun.runtimes.mounts.mount_s3,
+            AutoMountType.env: mlrun.runtimes.mounts.set_env_variables,
         }[self]
 
 
-class KubeResource(BaseRuntime, KfpAdapterMixin):
+class KubeResource(BaseRuntime):
     """
     A parent class for runtimes that generate k8s resources when executing.
     """
@@ -1026,7 +1056,8 @@ class KubeResource(BaseRuntime, KfpAdapterMixin):
 
     def get_env(self, name, default=None):
         """Get the pod environment variable for the given name, if not found return the default
-        If it's a scalar value, will return it, if the value is from source, return the k8s struct (V1EnvVarSource)"""
+        If it's a scalar value, will return it, if the value is from source, return the k8s struct (V1EnvVarSource)
+        """
         for env_var in self.spec.env:
             if get_item_name(env_var) == name:
                 # valueFrom is a workaround for now, for some reason the envs aren't getting sanitized
@@ -1080,7 +1111,10 @@ class KubeResource(BaseRuntime, KfpAdapterMixin):
             else:
                 raise mlrun.errors.MLRunNotFoundError(f"{file_path} does not exist")
         for name, value in env_vars.items():
-            self.set_env(name, value)
+            if isinstance(value, dict) and "valueFrom" in value:
+                self.set_env(name, value_from=value["valueFrom"])
+            else:
+                self.set_env(name, value)
         return self
 
     def set_image_pull_configuration(
@@ -1270,6 +1304,36 @@ class KubeResource(BaseRuntime, KfpAdapterMixin):
             )
         self.spec.security_context = security_context
 
+    def apply(
+        self,
+        modifier: typing.Callable[["KubeResource"], "KubeResource"],
+    ) -> "KubeResource":
+        """
+        Apply a modifier to the runtime which is used to change the runtimes k8s object's spec.
+        All modifiers accept Kube, apply some changes on its spec and return it so modifiers can be chained
+        one after the other.
+
+        :param modifier: a modifier callable object
+        :return: the runtime (self) after the modifications
+        """
+        modifier(self)
+        if AutoMountType.is_auto_modifier(modifier):
+            self.spec.disable_auto_mount = True
+
+        api_client = k8s_client.ApiClient()
+        if self.spec.env:
+            for index, env in enumerate(
+                api_client.sanitize_for_serialization(self.spec.env)
+            ):
+                self.spec.env[index] = env
+
+        if self.spec.volumes and self.spec.volume_mounts:
+            vols = api_client.sanitize_for_serialization(self.spec.volumes)
+            mounts = api_client.sanitize_for_serialization(self.spec.volume_mounts)
+            self.spec.update_vols_and_mounts(vols, mounts)
+
+        return self
+
     def list_valid_priority_class_names(self):
         return mlconf.get_valid_function_priority_class_names()
 

mlrun/runtimes/remotesparkjob.py

@@ -19,7 +19,7 @@ import kubernetes.client
 
 import mlrun.errors
 from mlrun.config import config
-from mlrun_pipelines.mounts import mount_v3io, mount_v3iod
+from mlrun.runtimes.mounts import mount_v3io, mount_v3iod
 
 from .kubejob import KubejobRuntime
 from .pod import KubeResourceSpec

mlrun/runtimes/sparkjob/spark3job.py

@@ -20,7 +20,7 @@ import mlrun.errors
 import mlrun.k8s_utils
 import mlrun.runtimes.pod
 from mlrun.config import config
-from mlrun_pipelines.mounts import mount_v3io, mount_v3iod
+from mlrun.runtimes.mounts import mount_v3io, mount_v3iod
 
 from ...execution import MLClientCtx
 from ...model import RunObject

mlrun/utils/clones.py

@@ -122,7 +122,7 @@ def add_credentials_git_remote_url(url: str, secrets=None) -> tuple[str, bool]:
         username, password = get_git_username_password_from_token(token)
 
     if username:
-        return f"https://{username}:{password}@{url_obj.hostname}{url_obj.path}", True
+        return f"https://{username}:{password}@{url_obj.netloc}{url_obj.path}", True
     return url, False
 
 

mlrun/utils/helpers.py

@@ -1265,14 +1265,24 @@ def datetime_to_iso(time_obj: Optional[datetime]) -> Optional[str]:
     return time_obj.isoformat()
 
 
-def enrich_datetime_with_tz_info(timestamp_string):
+def enrich_datetime_with_tz_info(timestamp_string) -> Optional[datetime]:
     if not timestamp_string:
         return timestamp_string
 
     if timestamp_string and not mlrun.utils.helpers.has_timezone(timestamp_string):
         timestamp_string += datetime.now(timezone.utc).astimezone().strftime("%z")
 
-    return datetime.strptime(timestamp_string, "%Y-%m-%d %H:%M:%S.%f%z")
+    for _format in [
+        # e.g: 2021-08-25 12:00:00.000Z
+        "%Y-%m-%d %H:%M:%S.%f%z",
+        # e.g: 2024-11-11 07:44:56+0000
+        "%Y-%m-%d %H:%M:%S%z",
+    ]:
+        try:
+            return datetime.strptime(timestamp_string, _format)
+        except ValueError as exc:
+            last_exc = exc
+    raise last_exc
 
 
 def has_timezone(timestamp):

mlrun/utils/logger.py

@@ -24,7 +24,7 @@ from traceback import format_exception
 from typing import IO, Optional, Union
 
 import orjson
-import pydantic
+import pydantic.v1
 
 from mlrun import errors
 from mlrun.config import config
@@ -33,7 +33,7 @@ from mlrun.config import config
 class _BaseFormatter(logging.Formatter):
     def _json_dump(self, json_object):
         def default(obj):
-            if isinstance(obj, pydantic.BaseModel):
+            if isinstance(obj, pydantic.v1.BaseModel):
                 return obj.dict()
 
             # EAFP all the way.

mlrun/utils/notifications/notification/__init__.py

@@ -14,30 +14,32 @@
 
 import enum
 
-from mlrun.common.schemas.notification import NotificationKind
-
-from .base import NotificationBase
-from .console import ConsoleNotification
-from .git import GitNotification
-from .ipython import IPythonNotification
-from .slack import SlackNotification
-from .webhook import WebhookNotification
+import mlrun.common.schemas.notification as notifications
+import mlrun.utils.notifications.notification.base as base
+import mlrun.utils.notifications.notification.console as console
+import mlrun.utils.notifications.notification.git as git
+import mlrun.utils.notifications.notification.ipython as ipython
+import mlrun.utils.notifications.notification.mail as mail
+import mlrun.utils.notifications.notification.slack as slack
+import mlrun.utils.notifications.notification.webhook as webhook
 
 
 class NotificationTypes(str, enum.Enum):
-    console = NotificationKind.console.value
-    git = NotificationKind.git.value
-    ipython = NotificationKind.ipython.value
-    slack = NotificationKind.slack.value
-    webhook = NotificationKind.webhook.value
+    console = notifications.NotificationKind.console.value
+    git = notifications.NotificationKind.git.value
+    ipython = notifications.NotificationKind.ipython.value
+    slack = notifications.NotificationKind.slack.value
+    mail = notifications.NotificationKind.mail.value
+    webhook = notifications.NotificationKind.webhook.value
 
-    def get_notification(self) -> type[NotificationBase]:
+    def get_notification(self) -> type[base.NotificationBase]:
         return {
-            self.console: ConsoleNotification,
-            self.git: GitNotification,
-            self.ipython: IPythonNotification,
-            self.slack: SlackNotification,
-            self.webhook: WebhookNotification,
+            self.console: console.ConsoleNotification,
+            self.git: git.GitNotification,
+            self.ipython: ipython.IPythonNotification,
+            self.slack: slack.SlackNotification,
+            self.mail: mail.MailNotification,
+            self.webhook: webhook.WebhookNotification,
         }.get(self)
 
     def inverse_dependencies(self) -> list[str]:
@@ -64,5 +66,6 @@ class NotificationTypes(str, enum.Enum):
             cls.git,
             cls.ipython,
             cls.slack,
+            cls.mail,
             cls.webhook,
         ]

mlrun/utils/notifications/notification/base.py

@@ -53,13 +53,13 @@ class NotificationBase:
     def push(
         self,
         message: str,
-        severity: typing.Union[
-            mlrun.common.schemas.NotificationSeverity, str
+        severity: typing.Optional[
+            typing.Union[mlrun.common.schemas.NotificationSeverity, str]
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
-        runs: typing.Union[mlrun.lists.RunList, list] = None,
-        custom_html: typing.Optional[str] = None,
-        alert: mlrun.common.schemas.AlertConfig = None,
-        event_data: mlrun.common.schemas.Event = None,
+        runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
+        custom_html: typing.Optional[typing.Optional[str]] = None,
+        alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
+        event_data: typing.Optional[mlrun.common.schemas.Event] = None,
     ):
         raise NotImplementedError()
 
@@ -81,13 +81,13 @@ class NotificationBase:
     def _get_html(
         self,
         message: str,
-        severity: typing.Union[
-            mlrun.common.schemas.NotificationSeverity, str
+        severity: typing.Optional[
+            typing.Union[mlrun.common.schemas.NotificationSeverity, str]
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
-        runs: typing.Union[mlrun.lists.RunList, list] = None,
-        custom_html: typing.Optional[str] = None,
-        alert: mlrun.common.schemas.AlertConfig = None,
-        event_data: mlrun.common.schemas.Event = None,
+        runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
+        custom_html: typing.Optional[typing.Optional[str]] = None,
+        alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
+        event_data: typing.Optional[mlrun.common.schemas.Event] = None,
     ) -> str:
         if custom_html:
             return custom_html

mlrun/utils/notifications/notification/console.py

@@ -31,13 +31,13 @@ class ConsoleNotification(NotificationBase):
     def push(
         self,
         message: str,
-        severity: typing.Union[
-            mlrun.common.schemas.NotificationSeverity, str
+        severity: typing.Optional[
+            typing.Union[mlrun.common.schemas.NotificationSeverity, str]
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
-        runs: typing.Union[mlrun.lists.RunList, list] = None,
-        custom_html: typing.Optional[str] = None,
-        alert: mlrun.common.schemas.AlertConfig = None,
-        event_data: mlrun.common.schemas.Event = None,
+        runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
+        custom_html: typing.Optional[typing.Optional[str]] = None,
+        alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
+        event_data: typing.Optional[mlrun.common.schemas.Event] = None,
     ):
         severity = self._resolve_severity(severity)
         print(f"[{severity}] {message}")

mlrun/utils/notifications/notification/git.py

@@ -54,13 +54,13 @@ class GitNotification(NotificationBase):
     async def push(
         self,
         message: str,
-        severity: typing.Union[
-            mlrun.common.schemas.NotificationSeverity, str
+        severity: typing.Optional[
+            typing.Union[mlrun.common.schemas.NotificationSeverity, str]
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
-        runs: typing.Union[mlrun.lists.RunList, list] = None,
-        custom_html: typing.Optional[str] = None,
-        alert: mlrun.common.schemas.AlertConfig = None,
-        event_data: mlrun.common.schemas.Event = None,
+        runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
+        custom_html: typing.Optional[typing.Optional[str]] = None,
+        alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
+        event_data: typing.Optional[mlrun.common.schemas.Event] = None,
     ):
         git_repo = self.params.get("repo", None)
         git_issue = self.params.get("issue", None)

mlrun/utils/notifications/notification/ipython.py

@@ -49,13 +49,13 @@ class IPythonNotification(NotificationBase):
     def push(
         self,
         message: str,
-        severity: typing.Union[
-            mlrun.common.schemas.NotificationSeverity, str
+        severity: typing.Optional[
+            typing.Union[mlrun.common.schemas.NotificationSeverity, str]
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
-        runs: typing.Union[mlrun.lists.RunList, list] = None,
-        custom_html: typing.Optional[str] = None,
-        alert: mlrun.common.schemas.AlertConfig = None,
-        event_data: mlrun.common.schemas.Event = None,
+        runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
+        custom_html: typing.Optional[typing.Optional[str]] = None,
+        alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
+        event_data: typing.Optional[mlrun.common.schemas.Event] = None,
     ):
         if not self._ipython:
             mlrun.utils.helpers.logger.debug(

mlrun/utils/notifications/notification/mail.py

@@ -0,0 +1,149 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+import typing
+from email.message import EmailMessage
+
+import aiosmtplib
+
+import mlrun.common.schemas
+import mlrun.lists
+import mlrun.utils.helpers
+import mlrun.utils.notifications.notification.base as base
+import mlrun.utils.regex
+
+DEFAULT_SMTP_PORT = 587
+
+
+class MailNotification(base.NotificationBase):
+    """
+    API/Client notification for sending run statuses as a mail message
+    """
+
+    boolean_params = ["use_tls", "start_tls", "validate_certs"]
+
+    required_params = [
+        "server_host",
+        "server_port",
+        "sender_address",
+        "username",
+        "password",
+        "email_addresses",
+    ] + boolean_params
+
+    @classmethod
+    def validate_params(cls, params):
+        for required_param in cls.required_params:
+            if required_param not in params:
+                raise ValueError(
+                    f"Parameter '{required_param}' is required for MailNotification"
+                )
+
+        for boolean_param in cls.boolean_params:
+            if not isinstance(params.get(boolean_param, None), bool):
+                raise ValueError(
+                    f"Parameter '{boolean_param}' must be a boolean for MailNotification"
+                )
+
+        cls._validate_emails(params)
+
+    async def push(
+        self,
+        message: str,
+        severity: typing.Optional[
+            typing.Union[mlrun.common.schemas.NotificationSeverity, str]
+        ] = mlrun.common.schemas.NotificationSeverity.INFO,
+        runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
+        custom_html: typing.Optional[typing.Optional[str]] = None,
+        alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
+        event_data: typing.Optional[mlrun.common.schemas.Event] = None,
+    ):
+        self.params.setdefault("subject", f"[{severity}] {message}")
+        self.params.setdefault("body", message)
+        await self._send_email(**self.params)
+
+    @classmethod
+    def enrich_default_params(
+        cls, params: dict, default_params: typing.Optional[dict] = None
+    ) -> dict:
+        params = super().enrich_default_params(params, default_params)
+        params.setdefault("use_tls", True)
+        params.setdefault("start_tls", False)
+        params.setdefault("validate_certs", True)
+        params.setdefault("server_port", DEFAULT_SMTP_PORT)
+
+        default_mail_address = params.pop("default_email_addresses", "")
+        email_addresses = params.get("email_addresses", default_mail_address)
+        if isinstance(email_addresses, list):
+            email_addresses = ",".join(email_addresses)
+        params["email_addresses"] = email_addresses
+
+        return params
+
+    @classmethod
+    def _validate_emails(cls, params):
+        cls._validate_email_address(params["sender_address"])
+
+        if not isinstance(params["email_addresses"], (str, list)):
+            raise ValueError(
+                "Parameter 'email_addresses' must be a string or a list of strings"
+            )
+
+        email_addresses = params["email_addresses"]
+        if isinstance(email_addresses, str):
+            email_addresses = email_addresses.split(",")
+        for email_address in email_addresses:
+            cls._validate_email_address(email_address)
+
+    @classmethod
+    def _validate_email_address(cls, email_address):
+        if not isinstance(email_address, str):
+            raise ValueError(f"Email address '{email_address}' must be a string")
+
+        if not re.match(mlrun.utils.regex.mail_regex, email_address):
+            raise ValueError(f"Invalid email address '{email_address}'")
+
+    @staticmethod
+    async def _send_email(
+        email_addresses: str,
+        sender_address: str,
+        server_host: str,
+        server_port: int,
+        username: str,
+        password: str,
+        use_tls: bool,
+        start_tls: bool,
+        validate_certs: bool,
+        subject: str,
+        body: str,
+        **kwargs,
+    ):
+        # Create the email message
+        message = EmailMessage()
+        message["From"] = sender_address
+        message["To"] = email_addresses
+        message["Subject"] = subject
+        message.set_content(body)
+
+        # Send the email
+        await aiosmtplib.send(
+            message,
+            hostname=server_host,
+            port=server_port,
+            username=username,
+            password=password,
+            use_tls=use_tls,
+            validate_certs=validate_certs,
+            start_tls=start_tls,
+        )

mlrun/utils/notifications/notification/slack.py

@@ -46,13 +46,13 @@ class SlackNotification(NotificationBase):
     async def push(
         self,
         message: str,
-        severity: typing.Union[
-            mlrun.common.schemas.NotificationSeverity, str
+        severity: typing.Optional[
+            typing.Union[mlrun.common.schemas.NotificationSeverity, str]
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
-        runs: typing.Union[mlrun.lists.RunList, list] = None,
-        custom_html: typing.Optional[str] = None,
-        alert: mlrun.common.schemas.AlertConfig = None,
-        event_data: mlrun.common.schemas.Event = None,
+        runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
+        custom_html: typing.Optional[typing.Optional[str]] = None,
+        alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
+        event_data: typing.Optional[mlrun.common.schemas.Event] = None,
     ):
         webhook = self.params.get("webhook", None) or mlrun.get_secret_or_env(
             "SLACK_WEBHOOK"

mlrun/utils/notifications/notification/webhook.py

@@ -37,13 +37,13 @@ class WebhookNotification(NotificationBase):
     async def push(
         self,
         message: str,
-        severity: typing.Union[
-            mlrun.common.schemas.NotificationSeverity, str
+        severity: typing.Optional[
+            typing.Union[mlrun.common.schemas.NotificationSeverity, str]
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
-        runs: typing.Union[mlrun.lists.RunList, list] = None,
-        custom_html: typing.Optional[str] = None,
-        alert: mlrun.common.schemas.AlertConfig = None,
-        event_data: mlrun.common.schemas.Event = None,
+        runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
+        custom_html: typing.Optional[typing.Optional[str]] = None,
+        alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
+        event_data: typing.Optional[mlrun.common.schemas.Event] = None,
     ):
         url = self.params.get("url", None)
         method = self.params.get("method", "post").lower()