mlrun 1.5.0rc11__py3-none-any.whl → 1.5.0rc13__py3-none-any.whl

mlrun/projects/project.py

@@ -38,7 +38,9 @@ import requests
 import yaml
 from deprecated import deprecated
 
+import mlrun.common.helpers
 import mlrun.common.schemas.model_monitoring
+import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.db
 import mlrun.errors
 import mlrun.runtimes
@@ -52,7 +54,12 @@ from ..artifacts.manager import ArtifactManager, dict_to_artifact, extend_artifa
 from ..datastore import store_manager
 from ..features import Feature
 from ..model import EntrypointParam, ImageBuilder, ModelObj
+from ..model_monitoring.application import (
+    ModelMonitoringApplication,
+    PushToMonitoringWriter,
+)
 from ..run import code_to_function, get_object, import_function, new_function
+from ..runtimes.function import RemoteRuntime
 from ..secrets import SecretsStore
 from ..utils import (
     is_ipython,
@@ -1767,6 +1774,102 @@ class MlrunProject(ModelObj):
         )
         return _run_project_setup(self, setup_file_path, save)
 
+    def set_model_monitoring_application(
+        self,
+        func: typing.Union[str, mlrun.runtimes.BaseRuntime] = None,
+        application_class: typing.Union[str, ModelMonitoringApplication] = None,
+        name: str = None,
+        image: str = None,
+        handler=None,
+        with_repo: bool = None,
+        tag: str = None,
+        requirements: typing.Union[str, typing.List[str]] = None,
+        requirements_file: str = "",
+        **application_kwargs,
+    ) -> mlrun.runtimes.BaseRuntime:
+        """
+        update or add a monitoring application to the project & deploy it.
+        examples::
+            project.set_model_monitoring_application(application_class_name="MyApp",
+                                                 image="davesh0812/mlrun-api:1.5.0", name="myApp")
+        :param func:                    function object or spec/code url, None refers to current Notebook
+        :param name:                    name of the function (under the project), can be specified with a tag to support
+                                        versions (e.g. myfunc:v1)
+                                        default: job
+        :param image:                   docker image to be used, can also be specified in
+                                        the function object/yaml
+        :param handler:                 default function handler to invoke (can only be set with .py/.ipynb files)
+        :param with_repo:               add (clone) the current repo to the build source
+        :param tag:                     function version tag (none for 'latest', can only be set with .py/.ipynb files)
+                                        if tag is specified and name is empty, the function key (under the project)
+                                        will be enriched with the tag value. (i.e. 'function-name:tag')
+        :param requirements:            a list of python packages
+        :param requirements_file:       path to a python requirements file
+        :param application_class:       Name or an Instance of a class that implementing the monitoring application.
+        :param application_kwargs:      Additional keyword arguments to be passed to the
+                                        monitoring application's constructor.
+        """
+
+        function_object: RemoteRuntime = None
+        kind = None
+        if (isinstance(func, str) or func is None) and application_class is not None:
+            kind = "serving"
+            func = mlrun.code_to_function(
+                name=name,
+                project=self.metadata.name,
+                tag=tag,
+                kind=kind,
+                image=image,
+                requirements=requirements,
+                requirements_file=requirements_file,
+            )
+            graph = func.set_topology("flow")
+            if isinstance(application_class, str):
+                first_step = graph.to(
+                    class_name=application_class, **application_kwargs
+                )
+            else:
+                first_step = graph.to(class_name=application_class)
+            first_step.to(
+                class_name=PushToMonitoringWriter(
+                    project=self.metadata.name,
+                    writer_application_name=mm_constants.MonitoringFunctionNames.WRITER,
+                    stream_uri=None,
+                ),
+            ).respond()
+        elif isinstance(func, str) and isinstance(handler, str):
+            kind = "nuclio"
+
+        resolved_function_name, function_object, func = self._resolved_function(
+            func,
+            name,
+            kind,
+            image,
+            handler,
+            with_repo,
+            tag,
+            requirements,
+            requirements_file,
+        )
+        models_names = "all"
+        function_object.set_label(
+            mm_constants.ModelMonitoringAppTag.KEY,
+            mm_constants.ModelMonitoringAppTag.VAL,
+        )
+        function_object.set_label("models", models_names)
+
+        if not mlrun.mlconf.is_ce_mode():
+            function_object.apply(mlrun.mount_v3io())
+        # Deploy & Add stream triggers
+        self.deploy_function(
+            function_object,
+        )
+
+        # save to project spec
+        self.spec.set_function(resolved_function_name, function_object, func)
+
+        return function_object
+
     def set_function(
         self,
         func: typing.Union[str, mlrun.runtimes.BaseRuntime] = None,
@@ -1805,29 +1908,56 @@ class MlrunProject(ModelObj):
             # by providing a path to a pip requirements file
             proj.set_function('my.py', requirements="requirements.txt")
 
-        :param func:                function object or spec/code url, None refers to current Notebook
-        :param name:                name of the function (under the project), can be specified with a tag to support
-                                    versions (e.g. myfunc:v1)
-        :param kind:                runtime kind e.g. job, nuclio, spark, dask, mpijob
-                                    default: job
-        :param image:               docker image to be used, can also be specified in
-                                    the function object/yaml
-        :param handler:             default function handler to invoke (can only be set with .py/.ipynb files)
-        :param with_repo:           add (clone) the current repo to the build source
-        :param tag:                 function version tag (none for 'latest', can only be set with .py/.ipynb files)
-                                    if tag is specified and name is empty, the function key (under the project)
-                                    will be enriched with the tag value. (i.e. 'function-name:tag')
-        :param requirements:        a list of python packages
-        :param requirements_file:   path to a python requirements file
+        :param func:                Function object or spec/code url, None refers to current Notebook
+        :param name:                Name of the function (under the project), can be specified with a tag to support
+                                    Versions (e.g. myfunc:v1). If the `tag` parameter is provided, the tag in the name
+                                    must match the tag parameter.
+                                    Specifying a tag in the name will update the project's tagged function (myfunc:v1)
+        :param kind:                Runtime kind e.g. job, nuclio, spark, dask, mpijob
+                                    Default: job
+        :param image:               Docker image to be used, can also be specified in the function object/yaml
+        :param handler:             Default function handler to invoke (can only be set with .py/.ipynb files)
+        :param with_repo:           Add (clone) the current repo to the build source
+        :param tag:                 Function version tag to set (none for current or 'latest')
+                                    Specifying a tag as a parameter will update the project's tagged function
+                                    (myfunc:v1) and the untagged function (myfunc)
+        :param requirements:        A list of python packages
+        :param requirements_file:   Path to a python requirements file
 
-        :returns: project object
+        :returns: function object
         """
+        resolved_function_name, function_object, func = self._resolved_function(
+            func,
+            name,
+            kind,
+            image,
+            handler,
+            with_repo,
+            tag,
+            requirements,
+            requirements_file,
+        )
+        self.spec.set_function(resolved_function_name, function_object, func)
+        return function_object
+
+    def _resolved_function(
+        self,
+        func: typing.Union[str, mlrun.runtimes.BaseRuntime] = None,
+        name: str = "",
+        kind: str = "",
+        image: str = None,
+        handler=None,
+        with_repo: bool = None,
+        tag: str = None,
+        requirements: typing.Union[str, typing.List[str]] = None,
+        requirements_file: str = "",
+    ):
         if func is None and not _has_module(handler, kind):
             # if function path is not provided and it is not a module (no ".")
             # use the current notebook as default
             if not is_ipython:
                 raise ValueError(
-                    "function path or module must be specified (when not running inside a Notebook)"
+                    "Function path or module must be specified (when not running inside a Notebook)"
                 )
             from IPython import get_ipython
 
@@ -1837,14 +1967,31 @@ class MlrunProject(ModelObj):
                 func = path.relpath(func, self.spec.context)
 
         func = func or ""
+
         name = mlrun.utils.normalize_name(name) if name else name
+        untagged_name = name
+        # validate tag in name if specified
+        if len(split_name := name.split(":")) == 2:
+            untagged_name, name_tag = split_name
+            if tag and name_tag and tag != name_tag:
+                raise ValueError(
+                    f"Tag parameter ({tag}) and tag in function name ({name}) must match"
+                )
+
+            tag = tag or name_tag
+        elif len(split_name) > 2:
+            raise ValueError(
+                f"Function name ({name}) must be in the format <name>:<tag> or <name>"
+            )
+
         if isinstance(func, str):
+
             # in hub or db functions name defaults to the function name
             if not name and not (func.startswith("db://") or func.startswith("hub://")):
-                raise ValueError("function name must be specified")
+                raise ValueError("Function name must be specified")
             function_dict = {
                 "url": func,
-                "name": name,
+                "name": untagged_name,
                 "kind": kind,
                 "image": image,
                 "handler": handler,
@@ -1857,13 +2004,14 @@ class MlrunProject(ModelObj):
                 func, self
             )
             func["name"] = resolved_function_name
+
         elif hasattr(func, "to_dict"):
             resolved_function_name, function_object = _init_function_from_obj(
-                func, self, name=name
+                func, self, name=untagged_name
             )
             if handler:
                 raise ValueError(
-                    "default handler cannot be set for existing function object"
+                    "Default handler cannot be set for existing function object"
                 )
             if image:
                 function_object.spec.image = image
@@ -1875,18 +2023,21 @@ class MlrunProject(ModelObj):
                     requirements, requirements_file=requirements_file
                 )
             if not resolved_function_name:
-                raise ValueError("function name must be specified")
+                raise ValueError("Function name must be specified")
         else:
-            raise ValueError("func must be a function url or object")
+            raise ValueError("'func' parameter must be a function url or object")
 
-        if tag and not resolved_function_name.endswith(f":{tag}"):
-            # Update the tagged key as well for consistency
-            self.spec.set_function(
-                f"{resolved_function_name}:{tag}", function_object, func
-            )
+        function_object.metadata.tag = tag or function_object.metadata.tag or "latest"
+        # resolved_function_name is the name without the tag or the actual function name if it was not specified
+        # if the name contains the tag we only update the tagged entry
+        # if the name doesn't contain the tag (or was not specified) we update both the tagged and untagged entries
+        # for consistency
+        name = name or resolved_function_name
+        if tag and not name.endswith(f":{tag}"):
+            self.spec.set_function(f"{name}:{tag}", function_object, func)
 
-        self.spec.set_function(resolved_function_name, function_object, func)
-        return function_object
+        self.spec.set_function(name, function_object, func)
+        return name, function_object, func
 
     def remove_function(self, name):
         """remove a function from a project
@@ -1895,6 +2046,24 @@ class MlrunProject(ModelObj):
         """
         self.spec.remove_function(name)
 
+    def remove_model_monitoring_application(self, name):
+        """remove a function from a project and from the db.
+
+        :param name: name of the function (under the project)
+        """
+        application = self.get_function(key=name)
+        if (
+            application.metadata.labels.get(mm_constants.ModelMonitoringAppTag.KEY)
+            == mm_constants.ModelMonitoringAppTag.VAL
+        ):
+            self.remove_function(name=name)
+            mlrun.db.get_run_db().delete_function(name=name.lower())
+            logger.info(f"{name} application has been removed from {self.name} project")
+        else:
+            raise logger.error(
+                f"There is no model monitoring application with {name} name"
+            )
+
     def get_function(
         self,
         key,
@@ -2208,12 +2377,16 @@ class MlrunProject(ModelObj):
         sync: bool = False,
         watch: bool = False,
         dirty: bool = False,
+        # TODO: deprecated, remove in 1.6.0
+        ttl: int = None,
         engine: str = None,
         local: bool = None,
         schedule: typing.Union[
             str, mlrun.common.schemas.ScheduleCronTrigger, bool
         ] = None,
         timeout: int = None,
+        # TODO: deprecated, remove in 1.6.0
+        overwrite: bool = False,
         source: str = None,
         cleanup_ttl: int = None,
     ) -> _PipelineRunStatus:
@@ -2233,6 +2406,8 @@ class MlrunProject(ModelObj):
         :param sync:      force functions sync before run
         :param watch:     wait for pipeline completion
         :param dirty:     allow running the workflow when the git repo is dirty
+        :param ttl:       pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
+                          workflow and all its resources are deleted) (deprecated, use cleanup_ttl instead)
         :param engine:    workflow engine running the workflow.
                           supported values are 'kfp' (default), 'local' or 'remote'.
                           for setting engine for remote running use 'remote:local' or 'remote:kfp'.
@@ -2243,6 +2418,8 @@ class MlrunProject(ModelObj):
                           https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron
                           for using the pre-defined workflow's schedule, set `schedule=True`
         :param timeout:   timeout in seconds to wait for pipeline completion (watch will be activated)
+        :param overwrite: (deprecated) replacing the schedule of the same workflow (under the same name) if exists
+                          with the new one.
         :param source:    remote source to use instead of the actual `project.spec.source` (used when engine is remote).
                           for other engines the source is to validate that the code is up-to-date
         :param cleanup_ttl:
@@ -2251,10 +2428,26 @@ class MlrunProject(ModelObj):
         :returns: run id
         """
 
+        if ttl:
+            warnings.warn(
+                "'ttl' is deprecated, use 'cleanup_ttl' instead. "
+                "This will be removed in 1.6.0",
+                # TODO: Remove this in 1.6.0
+                FutureWarning,
+            )
+
+        if overwrite:
+            warnings.warn(
+                "'overwrite' is deprecated, running a schedule is now an upsert operation. "
+                "This will be removed in 1.6.0",
+                # TODO: Remove this in 1.6.0
+                FutureWarning,
+            )
+
         arguments = arguments or {}
         need_repo = self.spec._need_repo()
         if self.spec.repo and self.spec.repo.is_dirty():
-            msg = "you seem to have uncommitted git changes, use .push()"
+            msg = "You seem to have uncommitted git changes, use .push()"
             if dirty or not need_repo:
                 logger.warning("WARNING!, " + msg)
             else:
@@ -2262,7 +2455,7 @@ class MlrunProject(ModelObj):
 
         if need_repo and self.spec.repo and not self.spec.source:
             raise ProjectError(
-                "remote repo is not defined, use .create_remote() + push()"
+                "Remote repo is not defined, use .create_remote() + push()"
             )
 
         self.sync_functions(always=sync)
@@ -2276,14 +2469,16 @@ class MlrunProject(ModelObj):
             if self.spec.workflows:
                 name = list(self.spec._workflows.keys())[0]
             else:
-                raise ValueError("workflow name or path must be specified")
+                raise ValueError("Workflow name or path must be specified")
 
         if workflow_path or (workflow_handler and callable(workflow_handler)):
             workflow_spec = WorkflowSpec(path=workflow_path, args=arguments)
         else:
             workflow_spec = self.spec._workflows[name].copy()
             workflow_spec.merge_args(arguments)
-        workflow_spec.cleanup_ttl = cleanup_ttl or workflow_spec.cleanup_ttl
+        workflow_spec.cleanup_ttl = (
+            cleanup_ttl or ttl or workflow_spec.cleanup_ttl or workflow_spec.ttl
+        )
         workflow_spec.run_local = local
 
         name = f"{self.metadata.name}-{name}" if name else self.metadata.name
@@ -2324,7 +2519,7 @@ class MlrunProject(ModelObj):
         if not workflow_spec.schedule:
             # Failure and schedule messages already logged
             logger.info(
-                f"started run workflow {name} with run id = '{run.run_id}' by {workflow_engine.engine} engine"
+                f"Started run workflow {name} with run id = '{run.run_id}' by {workflow_engine.engine} engine"
             )
         workflow_spec.clear_tmp()
         if (timeout or watch) and not workflow_spec.schedule:
@@ -2957,6 +3152,18 @@ class MlrunProject(ModelObj):
             # convert dict to function objects
             return [mlrun.new_function(runtime=func) for func in functions]
 
+    def list_model_monitoring_applications(self):
+        """Retrieve a list of alll the model monitoring application.
+        example::
+            functions = project.list_model_monitoring_applications()
+        :returns: List of function objects.
+        """
+        return self.list_functions(
+            labels=[
+                f"{mm_constants.ModelMonitoringAppTag.KEY}={mm_constants.ModelMonitoringAppTag.VAL}"
+            ]
+        )
+
     def list_runs(
         self,
         name: Optional[str] = None,
@@ -3113,31 +3320,32 @@ def _init_function_from_dict(
     has_module = _has_module(handler, kind)
     if not url and "spec" not in f and not has_module:
         # function must point to a file or a module or have a spec
-        raise ValueError("function missing a url or a spec or a module")
+        raise ValueError("Function missing a url or a spec or a module")
 
     relative_url = url
     url, in_context = project.get_item_absolute_path(url)
 
     if "spec" in f:
-        func = new_function(name, runtime=f["spec"])
+        func = new_function(name, runtime=f, tag=tag)
+
     elif not url and has_module:
         func = new_function(
             name, image=image, kind=kind or "job", handler=handler, tag=tag
         )
 
     elif is_yaml_path(url) or url.startswith("db://") or url.startswith("hub://"):
-        if tag:
-            raise ValueError(
-                "function with db:// or hub:// url or .yaml file, does not support tag value "
-            )
         func = import_function(url, new_name=name)
         if image:
             func.spec.image = image
+        if tag:
+            func.spec.tag = tag
+
     elif url.endswith(".ipynb"):
         # not defaulting kind to job here cause kind might come from magic annotations in the notebook
         func = code_to_function(
             name, filename=url, image=image, kind=kind, handler=handler, tag=tag
         )
+
     elif url.endswith(".py"):
         if not image and not project.default_image and kind != "local":
             raise ValueError(
@@ -3162,8 +3370,9 @@ def _init_function_from_dict(
                 handler=handler,
                 tag=tag,
             )
+
     else:
-        raise ValueError(f"unsupported function url:handler {url}:{handler} or no spec")
+        raise ValueError(f"Unsupported function url:handler {url}:{handler} or no spec")
 
     if with_repo:
         # mark source to be enriched before run with project source (enrich_function_object)
@@ -3171,7 +3380,7 @@ def _init_function_from_dict(
     if requirements:
         func.with_requirements(requirements)
 
-    return _init_function_from_obj(func, project, name)
+    return _init_function_from_obj(func, project)
 
 
 def _init_function_from_obj(
@@ -3190,9 +3399,14 @@ def _init_function_from_obj(
         build.code_origin = origin
     if project.metadata.name:
         func.metadata.project = project.metadata.name
+
+    # TODO: deprecate project tag
     if project.spec.tag:
         func.metadata.tag = project.spec.tag
-    return name or func.metadata.name, func
+
+    if name:
+        func.metadata.name = name
+    return func.metadata.name, func
 
 
 def _has_module(handler, kind):

mlrun/run.py

@@ -348,6 +348,7 @@ def get_or_create_ctx(
     rundb: str = "",
     project: str = "",
     upload_artifacts=False,
+    labels: dict = None,
 ):
     """called from within the user program to obtain a run context
 
@@ -366,7 +367,7 @@ def get_or_create_ctx(
     :param project:  project to initiate the context in (by default mlrun.mlctx.default_project)
     :param upload_artifacts:  when using local context (not as part of a job/run), upload artifacts to the
                               system default artifact path location
-
+    :param labels:      dict of the context labels
     :return: execution context
 
     Examples::
@@ -442,6 +443,9 @@ def get_or_create_ctx(
     ctx = MLClientCtx.from_dict(
         newspec, rundb=out, autocommit=autocommit, tmp=tmp, host=socket.gethostname()
     )
+    labels = labels or {}
+    for key, val in labels.items():
+        ctx.set_label(key=key, value=val)
     global_context.set(ctx)
     return ctx
 
@@ -679,7 +683,6 @@ def _process_runtime(command, runtime, kind):
     if not runtime:
         runtime = {}
     update_in(runtime, "spec.command", command)
-    runtime["kind"] = kind
     if kind != RuntimeKinds.remote:
         if command:
             update_in(runtime, "spec.command", command)

mlrun/runtimes/__init__.py

@@ -132,6 +132,8 @@ class RuntimeKinds(object):
             RuntimeKinds.mpijob,
             RuntimeKinds.databricks,
             RuntimeKinds.local,
+            RuntimeKinds.handler,
+            "",
         ]
 
     @staticmethod

mlrun/runtimes/function.py

@@ -558,8 +558,9 @@ class RemoteRuntime(KubeResource):
             self.metadata.tag = tag
 
         if dashboard:
+            # TODO: remove in 1.6.0
             warnings.warn(
-                "'dashboard' parameter is no longer supported on client side, "
+                "'dashboard' parameter is no longer supported on client side, and will be removed in 1.6.0."
                 "it is being configured through the MLRun API.",
             )
 

mlrun/utils/helpers.py

@@ -812,6 +812,18 @@ def new_pipe_metadata(
     return conf
 
 
+# TODO: remove in 1.6.0
+@deprecated(
+    version="1.3.0",
+    reason="'new_pipe_meta' will be removed in 1.6.0",
+    category=FutureWarning,
+)
+def new_pipe_meta(artifact_path=None, ttl=None, *args):
+    return new_pipe_metadata(
+        artifact_path=artifact_path, cleanup_ttl=ttl, op_transformers=args
+    )
+
+
 def _convert_python_package_version_to_image_tag(version: typing.Optional[str]):
     return (
         version.replace("+", "-").replace("0.0.0-", "") if version is not None else None

mlrun/utils/http.py

@@ -58,6 +58,9 @@ class HTTPSessionWithRetry(requests.Session):
         requests.exceptions.ConnectTimeout,
         requests.exceptions.ReadTimeout,
         urllib3.exceptions.ReadTimeoutError,
+        # may occur when connection breaks during the request.
+        requests.exceptions.ChunkedEncodingError,
+        urllib3.exceptions.InvalidChunkLength,
     ]
 
     def __init__(

mlrun/utils/notifications/notification_pusher.py

@@ -42,8 +42,16 @@ class NotificationPusher(object):
 
     def __init__(self, runs: typing.Union[mlrun.lists.RunList, list]):
         self._runs = runs
-        self._sync_notifications: typing.List[NotificationBase] = []
-        self._async_notifications: typing.List[NotificationBase] = []
+        self._sync_notifications: typing.List[
+            typing.Tuple[
+                NotificationBase, mlrun.model.RunObject, mlrun.model.Notification
+            ]
+        ] = []
+        self._async_notifications: typing.List[
+            typing.Tuple[
+                NotificationBase, mlrun.model.RunObject, mlrun.model.Notification
+            ]
+        ] = []
 
         for run in self._runs:
             if isinstance(run, dict):
@@ -74,11 +82,17 @@ class NotificationPusher(object):
 
         def _sync_push():
             for notification_data in self._sync_notifications:
-                self._push_notification_sync(
-                    notification_data[0],
-                    notification_data[1],
-                    notification_data[2],
-                )
+                try:
+                    self._push_notification_sync(
+                        notification_data[0],
+                        notification_data[1],
+                        notification_data[2],
+                    )
+                except Exception as exc:
+                    logger.warning(
+                        "Failed to push notification sync",
+                        error=mlrun.errors.err_to_str(exc),
+                    )
 
         async def _async_push():
             tasks = []
@@ -261,7 +275,7 @@ class NotificationPusher(object):
         update_notification_status_kwargs = {
             "run_uid": run.metadata.uid,
             "project": run.metadata.project,
-            "notification_object": notification_object,
+            "notification": notification_object,
             "status": mlrun.common.schemas.NotificationStatus.SENT,
         }
         try:

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "af4a00a1b624c2d68a13c74010dc09508ee935aa",
-  "version": "1.5.0-rc11"
+  "git_commit": "ec9859b42dcd9bac01f726f06b0c0d482572946a",
+  "version": "1.5.0-rc13"
 }