mlrun 1.3.1rc5__py3-none-any.whl → 1.4.0rc2__py3-none-any.whl

mlrun/runtimes/base.py

@@ -15,7 +15,6 @@ import enum
 import getpass
 import http
 import os.path
-import shlex
 import traceback
 import typing
 import uuid
@@ -33,8 +32,11 @@ from kubernetes.client.rest import ApiException
 from nuclio.build import mlrun_footer
 from sqlalchemy.orm import Session
 
+import mlrun.api.db.sqldb.session
+import mlrun.api.utils.singletons.db
 import mlrun.errors
 import mlrun.utils.helpers
+import mlrun.utils.notifications
 import mlrun.utils.regex
 from mlrun.api import schemas
 from mlrun.api.constants import LogSources
@@ -345,6 +347,7 @@ class BaseRuntime(ModelObj):
         local_code_path=None,
         auto_build=None,
         param_file_secrets: Dict[str, str] = None,
+        notifications: List[mlrun.model.Notification] = None,
         returns: Optional[List[Union[str, Dict[str, str]]]] = None,
     ) -> RunObject:
         """
@@ -379,6 +382,7 @@ class BaseRuntime(ModelObj):
                            function run, use only if you dont plan on changing the build config between runs
         :param param_file_secrets: dictionary of secrets to be used only for accessing the hyper-param parameter file.
                             These secrets are only used locally and will not be stored anywhere
+        :param notifications: list of notifications to push when the run is completed
         :param returns: List of log hints - configurations for how to log the returning values from the handler's run
                         (as artifacts or results). The list's length must be equal to the amount of returning objects. A
                         log hint may be given as:
@@ -408,7 +412,7 @@ class BaseRuntime(ModelObj):
                 raise mlrun.errors.MLRunInvalidArgumentError(
                     "local and schedule cannot be used together"
                 )
-            return self._run_local(
+            result = self._run_local(
                 run,
                 local_code_path,
                 project,
@@ -419,7 +423,10 @@ class BaseRuntime(ModelObj):
                 inputs,
                 returns,
                 artifact_path,
+                notifications=notifications,
             )
+            self._save_or_push_notifications(result, local)
+            return result
 
         run = self._enrich_run(
             run,
@@ -436,6 +443,7 @@ class BaseRuntime(ModelObj):
             out_path,
             artifact_path,
             workdir,
+            notifications,
         )
         self._validate_output_path(run)
         db = self._get_db()
@@ -533,6 +541,8 @@ class BaseRuntime(ModelObj):
                 last_err = err
                 result = self._update_run_state(task=run, err=err)
 
+        self._save_or_push_notifications(run)
+
         self._post_run(result, execution)  # hook for runtime specific cleanup
 
         return self._wrap_run_result(result, run, schedule=schedule, err=last_err)
@@ -630,6 +640,7 @@ class BaseRuntime(ModelObj):
         inputs,
         returns,
         artifact_path,
+        notifications: List[mlrun.model.Notification] = None,
     ):
         # allow local run simulation with a flip of a flag
         command = self
@@ -650,6 +661,7 @@ class BaseRuntime(ModelObj):
             artifact_path=artifact_path,
             mode=self.spec.mode,
             allow_empty_resources=self.spec.allow_empty_resources,
+            notifications=notifications,
             returns=returns,
         )
 
@@ -688,6 +700,7 @@ class BaseRuntime(ModelObj):
         out_path,
         artifact_path,
         workdir,
+        notifications: List[mlrun.model.Notification] = None,
     ):
         runspec.spec.handler = (
             handler or runspec.spec.handler or self.spec.default_handler or ""
@@ -789,6 +802,8 @@ class BaseRuntime(ModelObj):
             runspec.spec.output_path = mlrun.utils.helpers.fill_artifact_path_template(
                 runspec.spec.output_path, runspec.metadata.project
             )
+
+        runspec.spec.notifications = notifications or runspec.spec.notifications or []
         return runspec
 
     def _submit_job(self, run: RunObject, schedule, db, watch):
@@ -1049,6 +1064,47 @@ class BaseRuntime(ModelObj):
 
         return resp
 
+    def _save_or_push_notifications(self, runobj: RunObject, local: bool = False):
+
+        if not runobj.spec.notifications:
+            logger.debug(
+                "No notifications to push for run", run_uid=runobj.metadata.uid
+            )
+            return
+
+        # TODO: add support for other notifications per run iteration
+        if runobj.metadata.iteration and runobj.metadata.iteration > 0:
+            logger.debug(
+                "Notifications per iteration are not supported, skipping",
+                run_uid=runobj.metadata.uid,
+            )
+            return
+
+        # If the run is remote, and we are in the SDK, we let the api deal with the notifications
+        # so there's nothing to do here.
+        # Otherwise, we continue on.
+        if is_running_as_api():
+
+            # import here to avoid circular imports and to avoid importing api requirements
+            from mlrun.api.crud import Notifications
+
+            # If in the api server, we can assume that watch=False, so we save notification
+            # configs to the DB, for the run monitor to later pick up and push.
+            session = mlrun.api.db.sqldb.session.create_session()
+            Notifications().store_run_notifications(
+                session,
+                runobj.spec.notifications,
+                runobj.metadata.uid,
+                runobj.metadata.project,
+            )
+
+        elif local:
+            # If the run is local, we can assume that watch=True, therefore this code runs
+            # once the run is completed, and we can just push the notifications.
+            # TODO: add store_notifications API endpoint so we can store notifications pushed from the
+            #       SDK for documentation purposes.
+            mlrun.utils.notifications.NotificationPusher([runobj]).push()
+
     def _force_handler(self, handler):
         if not handler:
             raise RunError(f"handler must be provided for {self.kind} runtime")
@@ -1220,15 +1276,19 @@ class BaseRuntime(ModelObj):
         :param verify_base_image:  verify that the base image is configured
         :return: function object
         """
-        encoded_requirements = self._encode_requirements(requirements)
-        commands = self.spec.build.commands or [] if not overwrite else []
-        new_command = f"python -m pip install {encoded_requirements}"
-        # make sure we dont append the same line twice
-        if new_command not in commands:
-            commands.append(new_command)
-        self.spec.build.commands = commands
+        resolved_requirements = self._resolve_requirements(requirements)
+        requirements = self.spec.build.requirements or [] if not overwrite else []
+
+        # make sure we don't append the same line twice
+        for requirement in resolved_requirements:
+            if requirement not in requirements:
+                requirements.append(requirement)
+
+        self.spec.build.requirements = requirements
+
         if verify_base_image:
             self.verify_base_image()
+
         return self
 
     def with_commands(
@@ -1270,8 +1330,10 @@ class BaseRuntime(ModelObj):
 
     def verify_base_image(self):
         build = self.spec.build
-        require_build = build.commands or (
-            build.source and not build.load_source_on_run
+        require_build = (
+            build.commands
+            or build.requirements
+            or (build.source and not build.load_source_on_run)
         )
         image = self.spec.image
         # we allow users to not set an image, in that case we'll use the default
@@ -1396,15 +1458,16 @@ class BaseRuntime(ModelObj):
                             line += f", default={p['default']}"
                         print("    " + line)
 
-    def _encode_requirements(self, requirements_to_encode):
-
+    @staticmethod
+    def _resolve_requirements(requirements_to_resolve: typing.Union[str, list]) -> list:
         # if a string, read the file then encode
-        if isinstance(requirements_to_encode, str):
-            with open(requirements_to_encode, "r") as fp:
-                requirements_to_encode = fp.read().splitlines()
+        if isinstance(requirements_to_resolve, str):
+            with open(requirements_to_resolve, "r") as fp:
+                requirements_to_resolve = fp.read().splitlines()
 
         requirements = []
-        for requirement in requirements_to_encode:
+        for requirement in requirements_to_resolve:
+            # clean redundant leading and trailing whitespaces
             requirement = requirement.strip()
 
             # ignore empty lines
@@ -1417,21 +1480,9 @@ class BaseRuntime(ModelObj):
             if len(inline_comment) > 1:
                 requirement = inline_comment[0].strip()
 
-            # -r / --requirement are flags and should not be escaped
-            # we allow such flags (could be passed within the requirements.txt file) and do not
-            # try to open the file and include its content since it might be a remote file
-            # given on the base image.
-            for req_flag in ["-r", "--requirement"]:
-                if requirement.startswith(req_flag):
-                    requirement = requirement[len(req_flag) :].strip()
-                    requirements.append(req_flag)
-                    break
-
-            # wrap in single quote to ensure that the requirement is treated as a single string
-            # quote the requirement to avoid issues with special characters, double quotes, etc.
-            requirements.append(shlex.quote(requirement))
+            requirements.append(requirement)
 
-        return " ".join(requirements)
+        return requirements
 
     def _validate_output_path(self, run):
         if is_local(run.spec.output_path):

mlrun/runtimes/function.py

@@ -14,6 +14,7 @@
 
 import asyncio
 import json
+import shlex
 import typing
 import warnings
 from base64 import b64encode
@@ -1389,6 +1390,27 @@ def compile_function_config(
         config=function.spec.config,
     )
     nuclio_spec.cmd = function.spec.build.commands or []
+
+    if function.spec.build.requirements:
+        resolved_requirements = []
+        # wrap in single quote to ensure that the requirement is treated as a single string
+        # quote the requirement to avoid issues with special characters, double quotes, etc.
+        for requirement in function.spec.build.requirements:
+            # -r / --requirement are flags and should not be escaped
+            # we allow such flags (could be passed within the requirements.txt file) and do not
+            # try to open the file and include its content since it might be a remote file
+            # given on the base image.
+            for req_flag in ["-r", "--requirement"]:
+                if requirement.startswith(req_flag):
+                    requirement = requirement[len(req_flag) :].strip()
+                    resolved_requirements.append(req_flag)
+                    break
+
+            resolved_requirements.append(shlex.quote(requirement))
+
+        encoded_requirements = " ".join(resolved_requirements)
+        nuclio_spec.cmd.append(f"python -m pip install {encoded_requirements}")
+
     project = function.metadata.project or "default"
     tag = function.metadata.tag
     handler = function.spec.function_handler

mlrun/runtimes/kubejob.py

@@ -137,16 +137,12 @@ class KubejobRuntime(KubeResource):
             self.spec.build.image = image
         if base_image:
             self.spec.build.base_image = base_image
-        # if overwrite and requirements or commands passed, clear the existing commands
-        # (requirements are added to the commands parameter)
-        if (requirements or commands) and overwrite:
-            self.spec.build.commands = None
+        if commands:
+            self.with_commands(commands, overwrite=overwrite, verify_base_image=False)
         if requirements:
             self.with_requirements(
-                requirements, overwrite=False, verify_base_image=False
+                requirements, overwrite=overwrite, verify_base_image=False
             )
-        if commands:
-            self.with_commands(commands, overwrite=False, verify_base_image=False)
         if extra:
             self.spec.build.extra = extra
         if secret is not None:
@@ -198,7 +194,13 @@ class KubejobRuntime(KubeResource):
                     or "/mlrun/" in build.base_image
                 )
 
-        if not build.source and not build.commands and not build.extra and with_mlrun:
+        if (
+            not build.source
+            and not build.commands
+            and not build.requirements
+            and not build.extra
+            and with_mlrun
+        ):
             logger.info(
                 "running build to add mlrun package, set "
                 "with_mlrun=False to skip if its already in the image"

mlrun/runtimes/serving.py

@@ -319,7 +319,7 @@ class ServingRuntime(RemoteRuntime):
                                 example::
 
                                     # initialize a new serving function
-                                    serving_fn = mlrun.import_function("hub://v2_model_server", new_name="serving")
+                                    serving_fn = mlrun.import_function("hub://v2-model-server", new_name="serving")
                                     # apply model monitoring and set monitoring batch job to run every 3 hours
                                     tracking_policy = {'default_batch_intervals':"0 */3 * * *"}
                                     serving_fn.set_tracking(tracking_policy=tracking_policy)

mlrun/runtimes/sparkjob/__init__.py

@@ -15,5 +15,4 @@
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
 from .abstract import SparkRuntimeHandler
-from .spark2job import Spark2Runtime
 from .spark3job import Spark3Runtime

mlrun/runtimes/sparkjob/abstract.py

@@ -963,8 +963,6 @@ class SparkRuntimeHandler(BaseRuntimeHandler):
         """
         Handling config maps deletion
         """
-        if grace_period is None:
-            grace_period = config.runtime_resources_deletion_grace_period
         uids = []
         for crd_dict in deleted_resources:
             uid = crd_dict["metadata"].get("labels", {}).get("mlrun/uid", None)

mlrun/serving/states.py

@@ -28,7 +28,7 @@ from ..errors import MLRunInvalidArgumentError, err_to_str
 from ..model import ModelObj, ObjectDict
 from ..platforms.iguazio import parse_path
 from ..utils import get_class, get_function
-from .utils import _extract_input_data, _update_result_body
+from .utils import StepToDict, _extract_input_data, _update_result_body
 
 callable_prefix = "_"
 path_splitter = "/"
@@ -279,7 +279,7 @@ class BaseStep(ModelObj):
 
     def to(
         self,
-        class_name: Union[str, type] = None,
+        class_name: Union[str, StepToDict] = None,
         name: str = None,
         handler: str = None,
         graph_shape: str = None,

mlrun/utils/helpers.py

@@ -1000,7 +1000,7 @@ def create_class(pkg_class: str):
     return class_
 
 
-def create_function(pkg_func: list):
+def create_function(pkg_func: str):
     """Create a function from a package.module.function string
 
     :param pkg_func:  full function location,

mlrun/utils/notifications/notification/__init__.py

@@ -15,7 +15,7 @@
 import enum
 import typing
 
-from .base import NotificationBase, NotificationSeverity  # noqa
+from .base import NotificationBase
 from .console import ConsoleNotification
 from .git import GitNotification
 from .ipython import IPythonNotification

mlrun/utils/notifications/notification/base.py

@@ -12,35 +12,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import enum
 import typing
 
+import mlrun.api.schemas
 import mlrun.lists
 
 
-class NotificationSeverity(str, enum.Enum):
-    INFO = "info"
-    DEBUG = "debug"
-    VERBOSE = "verbose"
-    WARNING = "warning"
-    ERROR = "error"
-
-
 class NotificationBase:
     def __init__(
         self,
+        name: str = None,
         params: typing.Dict[str, str] = None,
     ):
+        self.name = name
         self.params = params or {}
 
     @property
     def active(self) -> bool:
         return True
 
-    def send(
+    def push(
         self,
         message: str,
-        severity: typing.Union[NotificationSeverity, str] = NotificationSeverity.INFO,
+        severity: typing.Union[
+            mlrun.api.schemas.NotificationSeverity, str
+        ] = mlrun.api.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
     ):
@@ -52,16 +48,21 @@ class NotificationBase:
     ) -> None:
         self.params = params or {}
 
-    @staticmethod
     def _get_html(
+        self,
         message: str,
-        severity: typing.Union[NotificationSeverity, str] = NotificationSeverity.INFO,
+        severity: typing.Union[
+            mlrun.api.schemas.NotificationSeverity, str
+        ] = mlrun.api.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
     ) -> str:
         if custom_html:
             return custom_html
 
+        if self.name:
+            message = f"{self.name}: {message}"
+
         if not runs:
             return f"[{severity}] {message}"
 

mlrun/utils/notifications/notification/console.py

@@ -16,10 +16,11 @@ import typing
 
 import tabulate
 
+import mlrun.api.schemas
 import mlrun.lists
 import mlrun.utils.helpers
 
-from .base import NotificationBase, NotificationSeverity
+from .base import NotificationBase
 
 
 class ConsoleNotification(NotificationBase):
@@ -27,10 +28,12 @@ class ConsoleNotification(NotificationBase):
     Client only notification for printing run status notifications in console
     """
 
-    def send(
+    def push(
         self,
         message: str,
-        severity: typing.Union[NotificationSeverity, str] = NotificationSeverity.INFO,
+        severity: typing.Union[
+            mlrun.api.schemas.NotificationSeverity, str
+        ] = mlrun.api.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
     ):

mlrun/utils/notifications/notification/git.py

@@ -16,12 +16,13 @@ import json
 import os
 import typing
 
-import requests
+import aiohttp
 
+import mlrun.api.schemas
 import mlrun.errors
 import mlrun.lists
 
-from .base import NotificationBase, NotificationSeverity
+from .base import NotificationBase
 
 
 class GitNotification(NotificationBase):
@@ -29,10 +30,12 @@ class GitNotification(NotificationBase):
     API/Client notification for setting a rich run statuses git issue comment (github/gitlab)
     """
 
-    def send(
+    async def push(
         self,
         message: str,
-        severity: typing.Union[NotificationSeverity, str] = NotificationSeverity.INFO,
+        severity: typing.Union[
+            mlrun.api.schemas.NotificationSeverity, str
+        ] = mlrun.api.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
     ):
@@ -45,7 +48,7 @@ class GitNotification(NotificationBase):
         )
         server = self.params.get("server", None)
         gitlab = self.params.get("gitlab", False)
-        self._pr_comment(
+        await self._pr_comment(
             self._get_html(message, severity, runs, custom_html),
             git_repo,
             git_issue,
@@ -55,7 +58,7 @@ class GitNotification(NotificationBase):
         )
 
     @staticmethod
-    def _pr_comment(
+    async def _pr_comment(
         message: str,
         repo: str = None,
         issue: int = None,
@@ -111,9 +114,13 @@ class GitNotification(NotificationBase):
                 "Authorization": f"token {token}",
             }
             url = f"https://{server}/repos/{repo}/issues/{issue}/comments"
-        resp = requests.post(url=url, json={"body": str(message)}, headers=headers)
-        if not resp.ok:
-            raise mlrun.errors.MLRunBadRequestError(
-                "Failed commenting on PR", response=resp.text, status=resp.status_code
-            )
-        return resp.json()["id"]
+
+        async with aiohttp.ClientSession() as session:
+            resp = await session.post(url, headers=headers, json={"body": message})
+            if not resp.ok:
+                resp_text = await resp.text()
+                raise mlrun.errors.MLRunBadRequestError(
+                    f"Failed commenting on PR: {resp_text}", status=resp.status
+                )
+            data = await resp.json()
+            return data.get("id")

mlrun/utils/notifications/notification/ipython.py

@@ -14,10 +14,11 @@
 
 import typing
 
+import mlrun.api.schemas
 import mlrun.lists
 import mlrun.utils.helpers
 
-from .base import NotificationBase, NotificationSeverity
+from .base import NotificationBase
 
 
 class IPythonNotification(NotificationBase):
@@ -45,10 +46,12 @@ class IPythonNotification(NotificationBase):
     def active(self) -> bool:
         return self._ipython is not None
 
-    def send(
+    def push(
         self,
         message: str,
-        severity: typing.Union[NotificationSeverity, str] = NotificationSeverity.INFO,
+        severity: typing.Union[
+            mlrun.api.schemas.NotificationSeverity, str
+        ] = mlrun.api.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
     ):

mlrun/utils/notifications/notification/slack.py

@@ -12,15 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import typing
 
-import requests
+import aiohttp
 
+import mlrun.api.schemas
 import mlrun.lists
 import mlrun.utils.helpers
 
-from .base import NotificationBase, NotificationSeverity
+from .base import NotificationBase
 
 
 class SlackNotification(NotificationBase):
@@ -34,10 +34,12 @@ class SlackNotification(NotificationBase):
         "error": ":x:",
     }
 
-    def send(
+    async def push(
         self,
         message: str,
-        severity: typing.Union[NotificationSeverity, str] = NotificationSeverity.INFO,
+        severity: typing.Union[
+            mlrun.api.schemas.NotificationSeverity, str
+        ] = mlrun.api.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
     ):
@@ -53,17 +55,16 @@ class SlackNotification(NotificationBase):
 
         data = self._generate_slack_data(message, severity, runs)
 
-        response = requests.post(
-            webhook,
-            data=json.dumps(data),
-            headers={"Content-Type": "application/json"},
-        )
-        response.raise_for_status()
+        async with aiohttp.ClientSession() as session:
+            async with session.post(webhook, json=data) as response:
+                response.raise_for_status()
 
     def _generate_slack_data(
         self,
         message: str,
-        severity: typing.Union[NotificationSeverity, str] = NotificationSeverity.INFO,
+        severity: typing.Union[
+            mlrun.api.schemas.NotificationSeverity, str
+        ] = mlrun.api.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
     ) -> dict:
         data = {