mlrun 1.5.0rc12__py3-none-any.whl → 1.5.0rc13__py3-none-any.whl

mlrun/__main__.py

@@ -17,6 +17,7 @@ import json
 import pathlib
 import socket
 import traceback
+import warnings
 from ast import literal_eval
 from base64 import b64decode, b64encode
 from os import environ, path, remove
@@ -445,7 +446,7 @@ def run(
 @click.option("--archive", "-a", default="", help="destination archive for code (tar)")
 @click.option("--silent", is_flag=True, help="do not show build logs")
 @click.option("--with-mlrun", is_flag=True, help="add MLRun package")
-@click.option("--db", default="", help="save run results to path or DB url")
+@click.option("--db", default="", help="save run results to DB url")
 @click.option(
     "--runtime", "-r", default="", help="function spec dict, for pipeline usage"
 )
@@ -591,6 +592,13 @@ def build(
     default="",
     help="path/url of function yaml or function " "yaml or db://<project>/<name>[:tag]",
 )
+# TODO: Remove in 1.6.0
+@click.option(
+    "--dashboard",
+    "-d",
+    default="",
+    help="Deprecated. Keep empty to allow auto-detect by MLRun API",
+)
 @click.option("--project", "-p", default="", help="project name")
 @click.option("--model", "-m", multiple=True, help="model name and path (name=path)")
 @click.option("--kind", "-k", default=None, help="runtime sub kind")
@@ -609,6 +617,7 @@ def deploy(
     spec,
     source,
     func_url,
+    dashboard,
     project,
     model,
     tag,
@@ -669,6 +678,14 @@ def deploy(
             function.set_env(k, v)
     function.verbose = verbose
 
+    if dashboard:
+        warnings.warn(
+            "'--dashboard' is deprecated in 1.3.0, and will be removed in 1.6.0, "
+            "Keep '--dashboard' value empty to allow auto-detection by MLRun API.",
+            # TODO: Remove in 1.6.0
+            FutureWarning,
+        )
+
     try:
         addr = function.deploy(project=project, tag=tag)
     except Exception as err:
@@ -917,7 +934,9 @@ def version():
 
 @main.command()
 @click.argument("uid", type=str)
-@click.option("--project", "-p", help="project name")
+@click.option(
+    "--project", "-p", help="project name (defaults to mlrun.mlconf.default_project)"
+)
 @click.option("--offset", type=int, default=0, help="byte offset")
 @click.option("--db", help="api and db service path/url")
 @click.option("--watch", "-w", is_flag=True, help="watch/follow log")
@@ -1004,6 +1023,14 @@ def logs(uid, project, offset, db, watch):
     "https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron."
     "For using the pre-defined workflow's schedule, set --schedule 'true'",
 )
+# TODO: Remove in 1.6.0
+@click.option(
+    "--overwrite-schedule",
+    "-os",
+    is_flag=True,
+    help="Deprecated (Saving schedules is now an upsert opertaion)."
+    " Overwrite a schedule when submitting a new one with the same name.",
+)
 @click.option(
     "--save-secrets",
     is_flag=True,
@@ -1043,6 +1070,7 @@ def project(
     timeout,
     schedule,
     notifications,
+    overwrite_schedule,
     save_secrets,
     save,
 ):
@@ -1132,6 +1160,7 @@ def project(
                 local=local,
                 schedule=schedule,
                 timeout=timeout,
+                overwrite=overwrite_schedule,
             )
         except Exception as err:
             print(traceback.format_exc())

mlrun/api/api/endpoints/functions.py

@@ -47,6 +47,7 @@ import mlrun.api.utils.singletons.project_member
 import mlrun.common.model_monitoring
 import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas
+import mlrun.common.schemas.model_monitoring.constants as mm_constants
 from mlrun.api.api import deps
 from mlrun.api.crud.secrets import Secrets, SecretsClientType
 from mlrun.api.utils.builder import build_runtime
@@ -729,35 +730,31 @@ def _build_function(
         launcher.enrich_runtime(runtime=fn, full=is_nuclio_runtime)
 
         fn.save(versioned=False)
-        if is_nuclio_runtime:
-            if fn.kind == RuntimeKinds.serving:
-                # Handle model monitoring
+        if fn.kind in RuntimeKinds.nuclio_runtimes():
+            mlrun.api.api.utils.apply_enrichment_and_validation_on_function(
+                fn,
+                auth_info,
+            )
+            monitoring_application = (
+                fn.metadata.labels.get(mm_constants.ModelMonitoringAppTag.KEY)
+                == mm_constants.ModelMonitoringAppTag.VAL
+            )
+            serving_to_monitor = (
+                fn.kind == RuntimeKinds.serving and fn.spec.track_models
+            )
+            if serving_to_monitor or monitoring_application:
                 try:
-                    if fn.spec.track_models:
-                        logger.info("Tracking enabled, initializing model monitoring")
-
-                        # Generating model monitoring access key
+                    if not mlrun.mlconf.is_ce_mode():
+                        model_monitoring_access_key = process_model_monitoring_secret(
+                            db_session,
+                            fn.metadata.project,
+                            mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ACCESS_KEY,
+                        )
+                    else:
                         model_monitoring_access_key = None
-                        if not mlrun.mlconf.is_ce_mode():
-                            model_monitoring_access_key = process_model_monitoring_secret(
-                                db_session,
-                                fn.metadata.project,
-                                mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ACCESS_KEY,
-                            )
-
-                            stream_path = (
-                                mlrun.api.crud.model_monitoring.get_stream_path(
-                                    project=fn.metadata.project
-                                )
-                            )
-
-                            if stream_path.startswith("v3io://"):
-                                # Initialize model monitoring V3IO stream
-                                _create_model_monitoring_stream(
-                                    project=fn.metadata.project,
-                                    function=fn,
-                                    stream_path=stream_path,
-                                )
+                    if serving_to_monitor:
+                        # Handle model monitoring
+                        logger.info("Tracking enabled, initializing model monitoring")
 
                         if fn.spec.tracking_policy:
                             # Convert to `TrackingPolicy` object as `fn.spec.tracking_policy` is provided as a dict
@@ -768,7 +765,32 @@ def _build_function(
                             # Initialize tracking policy with default values
                             fn.spec.tracking_policy = TrackingPolicy()
 
-                        # deploy both model monitoring stream and model monitoring batch job
+                        if not mlrun.mlconf.is_ce_mode():
+                            # create v3io stream for model_monitoring_stream
+                            _create_model_monitoring_stream(
+                                project=fn.metadata.project,
+                                function=fn,
+                                monitoring_application=monitoring_application,
+                                stream_path=mlrun.api.crud.model_monitoring.get_stream_path(
+                                    project=fn.metadata.project,
+                                    application_name=mm_constants.MonitoringFunctionNames.STREAM,
+                                ),
+                            )
+                            if fn.spec.tracking_policy.application_batch:
+                                # create v3io stream for  model_monitoring_writer | model monitoring application
+                                _create_model_monitoring_stream(
+                                    project=fn.metadata.project,
+                                    function=fn,
+                                    monitoring_application=monitoring_application,
+                                    stream_path=mlrun.api.crud.model_monitoring.get_stream_path(
+                                        project=fn.metadata.project,
+                                        application_name=mm_constants.MonitoringFunctionNames.WRITER,
+                                    ),
+                                    access_key=model_monitoring_access_key,
+                                )
+
+                        # deploy model monitoring stream, model monitoring batch job,
+                        # model monitoring batch application job and model monitoring writer
                         mlrun.api.crud.model_monitoring.deployment.MonitoringDeployment().deploy_monitoring_functions(
                             project=fn.metadata.project,
                             db_session=db_session,
@@ -776,9 +798,32 @@ def _build_function(
                             tracking_policy=fn.spec.tracking_policy,
                             model_monitoring_access_key=model_monitoring_access_key,
                         )
+
+                    if monitoring_application:
+                        if not mlrun.mlconf.is_ce_mode():
+                            # create v3io stream for model monitoring application
+                            _create_model_monitoring_stream(
+                                project=fn.metadata.project,
+                                function=fn,
+                                monitoring_application=monitoring_application,
+                                stream_path=mlrun.api.crud.model_monitoring.get_stream_path(
+                                    project=fn.metadata.project,
+                                    application_name=fn.metadata.name,
+                                ),
+                                access_key=model_monitoring_access_key,
+                            )
+                        # apply stream trigger to monitoring application
+                        fn = mlrun.api.crud.model_monitoring.deployment.MonitoringDeployment()._apply_stream_trigger(
+                            project=fn.metadata.project,
+                            function=fn,
+                            model_monitoring_access_key=model_monitoring_access_key,
+                            function_name=fn.metadata.name,
+                            auth_info=auth_info,
+                        )
                 except Exception as exc:
                     logger.warning(
-                        "Failed deploying model monitoring infrastructure for the project",
+                        f"Failed deploying model monitoring infrastructure for the "
+                        f"{'project' if serving_to_monitor else f'{fn.metadata.name} application'}",
                         project=fn.metadata.project,
                         exc=exc,
                         traceback=traceback.format_exc(),
@@ -915,33 +960,46 @@ async def _get_function_status(data, auth_info: mlrun.common.schemas.AuthInfo):
         )
 
 
-def _create_model_monitoring_stream(project: str, function, stream_path):
-    _init_serving_function_stream_args(fn=function)
+def _create_model_monitoring_stream(
+    project: str,
+    function,
+    monitoring_application: bool,
+    stream_path: str,
+    access_key: str = None,
+):
+    if stream_path.startswith("v3io://"):
+        _init_serving_function_stream_args(fn=function)
 
-    _, container, stream_path = parse_model_endpoint_store_prefix(stream_path)
+        _, container, stream_path = parse_model_endpoint_store_prefix(stream_path)
 
-    # TODO: How should we configure sharding here?
-    logger.info(
-        "Creating model endpoint stream for project",
-        project=project,
-        stream_path=stream_path,
-        container=container,
-        endpoint=config.v3io_api,
-    )
+        # TODO: How should we configure sharding here?
+        logger.info(
+            "Creating model endpoint stream for project",
+            project=project,
+            stream_path=stream_path,
+            container=container,
+            endpoint=config.v3io_api,
+        )
 
-    v3io_client = v3io.dataplane.Client(
-        endpoint=config.v3io_api, access_key=os.environ.get("V3IO_ACCESS_KEY")
-    )
-    response = v3io_client.create_stream(
-        container=container,
-        path=stream_path,
-        shard_count=config.model_endpoint_monitoring.serving_stream_args.shard_count,
-        retention_period_hours=config.model_endpoint_monitoring.serving_stream_args.retention_period_hours,
-        raise_for_status=v3io.dataplane.RaiseForStatus.never,
-    )
+        v3io_client = v3io.dataplane.Client(
+            endpoint=config.v3io_api, access_key=os.environ.get("V3IO_ACCESS_KEY")
+        )
+        stream_args = (
+            config.model_endpoint_monitoring.application_stream_args
+            if monitoring_application
+            else config.model_endpoint_monitoring.serving_stream_args
+        )
+        response = v3io_client.create_stream(
+            container=container,
+            path=stream_path,
+            shard_count=stream_args.shard_count,
+            retention_period_hours=stream_args.retention_period_hours,
+            raise_for_status=v3io.dataplane.RaiseForStatus.never,
+            access_key=access_key,
+        )
 
-    if not (response.status_code == 400 and "ResourceInUse" in str(response.body)):
-        response.raise_for_status([409, 204])
+        if not (response.status_code == 400 and "ResourceInUse" in str(response.body)):
+            response.raise_for_status([409, 204])
 
 
 def _init_serving_function_stream_args(fn: ServingRuntime):