mlrun 1.6.0rc6__py3-none-any.whl → 1.6.0rc8__py3-none-any.whl

mlrun/projects/project.py

@@ -87,6 +87,7 @@ from .pipelines import (
     FunctionsDict,
     WorkflowSpec,
     _PipelineRunStatus,
+    _RemoteRunner,
     enrich_function_object,
     get_db_function,
     get_workflow_engine,
@@ -102,9 +103,9 @@ def init_repo(context, url, init_git):
     repo = None
     context_path = pathlib.Path(context)
     if not context_path.exists():
-        context_path.mkdir(parents=True)
+        context_path.mkdir(parents=True, exist_ok=True)
     elif not context_path.is_dir():
-        raise ValueError(f"context {context} is not a dir path")
+        raise ValueError(f"Context {context} is not a dir path")
     try:
         repo = git.Repo(context)
         url = get_repo_url(repo)
@@ -221,6 +222,7 @@ def new_project(
     if remote and url != remote:
         project.create_remote(remote)
     elif url:
+        logger.info("Identified pre-initialized git repo, using it", url=url)
         project.spec._source = url
         project.spec.origin_url = url
     if description:
@@ -366,7 +368,7 @@ def load_project(
         project = _load_project_dir(context, name, subpath)
 
     if not project.metadata.name:
-        raise ValueError("project name must be specified")
+        raise ValueError("Project name must be specified")
 
     if parameters:
         # Enable setting project parameters at load time, can be used to customize the project_setup
@@ -1593,7 +1595,7 @@ class MlrunProject(ModelObj):
         :param parameters:      key/value dict of model parameters
         :param inputs:          ordered list of model input features (name, type, ..)
         :param outputs:         ordered list of model output/result elements (name, type, ..)
-        :param upload:          upload to datastore (default is True)
+        :param upload:          upload to datastore (if not specified, defaults to True (uploads artifact))
         :param labels:          a set of key/value labels to tag the artifact with
         :param feature_vector:  feature store feature vector uri (store://feature-vectors/<project>/<name>[:tag])
         :param feature_weights: list of feature weights, one per input column
@@ -2499,6 +2501,7 @@ class MlrunProject(ModelObj):
         timeout: int = None,
         source: str = None,
         cleanup_ttl: int = None,
+        notifications: typing.List[mlrun.model.Notification] = None,
     ) -> _PipelineRunStatus:
         """run a workflow using kubeflow pipelines
 
@@ -2531,6 +2534,8 @@ class MlrunProject(ModelObj):
         :param cleanup_ttl:
                           pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
                           workflow and all its resources are deleted)
+        :param notifications:
+                          list of notifications to send for workflow completion
         :returns: run id
         """
 
@@ -2598,6 +2603,7 @@ class MlrunProject(ModelObj):
             artifact_path=artifact_path,
             namespace=namespace,
             source=source,
+            notifications=notifications,
         )
         # run is None when scheduling
         if run and run.state == mlrun.run.RunStatuses.failed:
@@ -2609,7 +2615,14 @@ class MlrunProject(ModelObj):
             )
         workflow_spec.clear_tmp()
         if (timeout or watch) and not workflow_spec.schedule:
-            run._engine.get_run_status(project=self, run=run, timeout=timeout)
+            # run's engine gets replaced with inner engine if engine is remote,
+            # so in that case we need to get the status from the remote engine manually
+            if engine == "remote":
+                status_engine = _RemoteRunner
+            else:
+                status_engine = run._engine
+
+            status_engine.get_run_status(project=self, run=run, timeout=timeout)
         return run
 
     def save_workflow(self, name, target, artifact_path=None, ttl=None):
@@ -2711,6 +2724,7 @@ class MlrunProject(ModelObj):
         """Set the credentials that will be used by the project's model monitoring
         infrastructure functions.
 
+        :param access_key:                Model Monitoring access key for managing user permissions
         :param access_key:                Model Monitoring access key for managing user permissions
         :param endpoint_store_connection: Endpoint store connection string
         :param stream_path:               Path to the model monitoring stream
@@ -2995,7 +3009,7 @@ class MlrunProject(ModelObj):
             * True: The existing params are replaced by the new ones
         :param extra_args:  A string containing additional builder arguments in the format of command-line options,
             e.g. extra_args="--skip-tls-verify --build-arg A=val"r
-        :param force_build:
+        :param force_build: set True for force building the image
         """
 
         if skip_deployed:
@@ -3228,13 +3242,16 @@ class MlrunProject(ModelObj):
         :param labels: Return functions that have specific labels assigned to them.
         :returns: List of function objects.
         """
-        labels = labels or []
+
+        model_monitoring_labels_list = [
+            f"{mm_constants.ModelMonitoringAppLabel.KEY}={mm_constants.ModelMonitoringAppLabel.VAL}"
+        ]
+        if labels:
+            model_monitoring_labels_list += labels
         return self.list_functions(
             name=name,
             tag=tag,
-            labels=[
-                f"{mm_constants.ModelMonitoringAppLabel.KEY}={mm_constants.ModelMonitoringAppLabel.VAL}"
-            ].extend(labels),
+            labels=model_monitoring_labels_list,
         )
 
     def list_runs(
@@ -3463,7 +3480,13 @@ def _init_function_from_dict(
         )
 
     elif url.endswith(".py"):
-        if not image and not project.default_image and kind != "local":
+        # when load_source_on_run is used we allow not providing image as code will be loaded pre-run. ML-4994
+        if (
+            not image
+            and not project.default_image
+            and kind != "local"
+            and not project.spec.load_source_on_run
+        ):
             raise ValueError(
                 "image must be provided with py code files which do not "
                 "run on 'local' engine kind"

mlrun/runtimes/__init__.py

@@ -199,6 +199,12 @@ class RuntimeKinds(object):
             return True
         return False
 
+    @staticmethod
+    def requires_image_name_for_execution(kind):
+        if RuntimeKinds.is_local_runtime(kind):
+            return False
+        return kind not in [RuntimeKinds.spark]
+
 
 def get_runtime_class(kind: str):
     if kind == RuntimeKinds.mpijob:

mlrun/runtimes/base.py

@@ -196,6 +196,9 @@ class BaseRuntime(ModelObj):
         self.metadata.labels[key] = str(value)
         return self
 
+    def set_categories(self, categories: List[str]):
+        self.metadata.categories = mlrun.utils.helpers.as_list(categories)
+
     @property
     def uri(self):
         return self._function_uri()
@@ -356,7 +359,7 @@ class BaseRuntime(ModelObj):
                           type can be given there. The artifact key must appear in the dictionary as "key": "the_key".
         :param state_thresholds:    Dictionary of states to time thresholds. The state will be matched against the
                 pod's status. The threshold should be a time string that conforms to timelength python package
-                standards and is at least 1 second (-1 for infinite).
+                standards and is at least 1 minute (-1 for infinite).
                 If the phase is active for longer than the threshold, the run will be aborted.
                 See mlconf.function.spec.state_thresholds for the state options and default values.
         :return: Run context object (RunObject) with run metadata, results and status
@@ -797,6 +800,14 @@ class BaseRuntime(ModelObj):
         self.spec.build = {}
         return self
 
+    def requires_build(self) -> bool:
+        build = self.spec.build
+        return (
+            build.commands
+            or build.requirements
+            or (build.source and not build.load_source_on_run)
+        )
+
     def prepare_image_for_deploy(self):
         """
         if a function has a 'spec.image' it is considered to be deployed,

mlrun/runtimes/daskjob.py

@@ -16,6 +16,7 @@ import inspect
 import socket
 import time
 from os import environ
+from typing import Callable, Dict, List, Optional, Union
 
 import mlrun.common.schemas
 import mlrun.errors
@@ -230,7 +231,7 @@ class DaskCluster(KubejobRuntime):
             if db_func and "status" in db_func:
                 self.status = db_func["status"]
                 if self.kfp:
-                    logger.info(f"dask status: {db_func['status']}")
+                    logger.info(f"Dask status: {db_func['status']}")
                 return "scheduler_address" in db_func["status"]
 
         return False
@@ -311,7 +312,7 @@ class DaskCluster(KubejobRuntime):
             if self.spec.service_type == "NodePort":
                 dash = f"{config.remote_host}:{self.status.node_ports.get('dashboard')}"
             else:
-                logger.info("to get a dashboard link, use NodePort service_type")
+                logger.info("To get a dashboard link, use NodePort service_type")
 
         return addr, dash
 
@@ -325,12 +326,12 @@ class DaskCluster(KubejobRuntime):
 
         if self.status.scheduler_address:
             addr, dash = self._remote_addresses()
-            logger.info(f"trying dask client at: {addr}")
+            logger.info(f"Trying dask client at: {addr}")
             try:
                 client = Client(addr)
             except OSError as exc:
                 logger.warning(
-                    f"remote scheduler at {addr} not ready, will try to restart {err_to_str(exc)}"
+                    f"Remote scheduler at {addr} not ready, will try to restart {err_to_str(exc)}"
                 )
 
                 status = self.get_status()
@@ -340,7 +341,7 @@ class DaskCluster(KubejobRuntime):
                 client = Client(addr)
 
             logger.info(
-                f"using remote dask scheduler ({self.status.cluster_name}) at: {addr}"
+                f"Using remote dask scheduler ({self.status.cluster_name}) at: {addr}"
             )
             if dash:
                 ipython_display(
@@ -457,6 +458,73 @@ class DaskCluster(KubejobRuntime):
         """
         self.spec._verify_and_set_requests("worker_resources", mem, cpu, patch=patch)
 
+    def set_state_thresholds(
+        self,
+        state_thresholds: Dict[str, str],
+        patch: bool = True,
+    ):
+        raise NotImplementedError(
+            "State thresholds is not supported for Dask runtime yet, use spec.scheduler_timeout instead.",
+        )
+
+    def run(
+        self,
+        runspec: Optional[
+            Union["mlrun.run.RunTemplate", "mlrun.run.RunObject", dict]
+        ] = None,
+        handler: Optional[Union[str, Callable]] = None,
+        name: Optional[str] = "",
+        project: Optional[str] = "",
+        params: Optional[dict] = None,
+        inputs: Optional[Dict[str, str]] = None,
+        out_path: Optional[str] = "",
+        workdir: Optional[str] = "",
+        artifact_path: Optional[str] = "",
+        watch: Optional[bool] = True,
+        schedule: Optional[Union[str, mlrun.common.schemas.ScheduleCronTrigger]] = None,
+        hyperparams: Optional[Dict[str, list]] = None,
+        hyper_param_options: Optional[mlrun.model.HyperParamOptions] = None,
+        verbose: Optional[bool] = None,
+        scrape_metrics: Optional[bool] = None,
+        local: Optional[bool] = False,
+        local_code_path: Optional[str] = None,
+        auto_build: Optional[bool] = None,
+        param_file_secrets: Optional[Dict[str, str]] = None,
+        notifications: Optional[List[mlrun.model.Notification]] = None,
+        returns: Optional[List[Union[str, Dict[str, str]]]] = None,
+        state_thresholds: Optional[Dict[str, int]] = None,
+        **launcher_kwargs,
+    ) -> RunObject:
+        if state_thresholds:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "State thresholds is not supported for Dask runtime yet, use spec.scheduler_timeout instead."
+            )
+        return super().run(
+            runspec=runspec,
+            handler=handler,
+            name=name,
+            project=project,
+            params=params,
+            inputs=inputs,
+            out_path=out_path,
+            workdir=workdir,
+            artifact_path=artifact_path,
+            watch=watch,
+            schedule=schedule,
+            hyperparams=hyperparams,
+            hyper_param_options=hyper_param_options,
+            verbose=verbose,
+            scrape_metrics=scrape_metrics,
+            local=local,
+            local_code_path=local_code_path,
+            auto_build=auto_build,
+            param_file_secrets=param_file_secrets,
+            notifications=notifications,
+            returns=returns,
+            state_thresholds=state_thresholds,
+            **launcher_kwargs,
+        )
+
     def _run(self, runobj: RunObject, execution):
 
         handler = runobj.spec.handler

mlrun/runtimes/databricks_job/databricks_runtime.py

@@ -127,6 +127,7 @@ if result:
         param_file_secrets: Optional[Dict[str, str]] = None,
         notifications: Optional[List[mlrun.model.Notification]] = None,
         returns: Optional[List[Union[str, Dict[str, str]]]] = None,
+        state_thresholds: Optional[Dict[str, int]] = None,
         **launcher_kwargs,
     ) -> RunObject:
         if local:
@@ -153,6 +154,7 @@ if result:
             param_file_secrets=param_file_secrets,
             notifications=notifications,
             returns=returns,
+            state_thresholds=state_thresholds,
             **launcher_kwargs,
         )
 

mlrun/runtimes/function.py

@@ -120,6 +120,7 @@ class NuclioSpec(KubeResourceSpec):
         "base_image_pull",
         "service_type",
         "add_templated_ingress_host_mode",
+        "disable_default_http_trigger",
     ]
 
     def __init__(
@@ -163,6 +164,7 @@ class NuclioSpec(KubeResourceSpec):
         add_templated_ingress_host_mode=None,
         clone_target_dir=None,
         state_thresholds=None,
+        disable_default_http_trigger=None,
     ):
 
         super().__init__(
@@ -210,6 +212,8 @@ class NuclioSpec(KubeResourceSpec):
         self.min_replicas = min_replicas or 1
         self.max_replicas = max_replicas or 4
 
+        self.disable_default_http_trigger = disable_default_http_trigger
+
         # When True it will set Nuclio spec.noBaseImagesPull to False (negative logic)
         # indicate that the base image should be pulled from the container registry (not cached)
         self.base_image_pull = False
@@ -413,6 +417,11 @@ class RemoteRuntime(KubeResource):
         :param extra_attributes: key/value dict of extra nuclio trigger attributes
         :return: function object (self)
         """
+        if self.disable_default_http_trigger:
+            logger.warning(
+                "Adding HTTP trigger despite the default HTTP trigger creation being disabled"
+            )
+
         annotations = annotations or {}
         if worker_timeout:
             gateway_timeout = gateway_timeout or (worker_timeout + 60)
@@ -683,6 +692,24 @@ class RemoteRuntime(KubeResource):
             "State thresholds do not apply for nuclio as it has its own function pods healthiness monitoring"
         )
 
+    @min_nuclio_versions("1.12.8")
+    def disable_default_http_trigger(
+        self,
+    ):
+        """
+        Disables nuclio's default http trigger creation
+        """
+        self.spec.disable_default_http_trigger = True
+
+    @min_nuclio_versions("1.12.8")
+    def enable_default_http_trigger(
+        self,
+    ):
+        """
+        Enables nuclio's default http trigger creation
+        """
+        self.spec.disable_default_http_trigger = False
+
     def _get_state(
         self,
         dashboard="",
@@ -876,11 +903,22 @@ class RemoteRuntime(KubeResource):
 
         if "://" not in path:
             if not self.status.address:
-                state, _, _ = self._get_state(dashboard, auth_info=auth_info)
-                if state != "ready" or not self.status.address:
-                    raise ValueError(
-                        "no function address or not ready, first run .deploy()"
+                # here we check that if default http trigger is disabled, function contains a custom http trigger
+                # Otherwise, the function is not invokable, so we raise an error
+                if (
+                    not self._trigger_of_kind_exists(kind="http")
+                    and self.spec.disable_default_http_trigger
+                ):
+                    raise mlrun.errors.MLRunPreconditionFailedError(
+                        "Default http trigger creation is disabled and there is no any other custom http trigger, "
+                        "so function can not be invoked via http. Either enable default http trigger creation or create"
+                        "custom http trigger"
                     )
+                state, _, _ = self._get_state(dashboard, auth_info=auth_info)
+                if state not in ["ready", "scaledToZero"]:
+                    logger.warning(f"Function is in the {state} state")
+                if not self.status.address:
+                    raise ValueError("no function address first run .deploy()")
 
             path = self._resolve_invocation_url(path, force_external_address)
 
@@ -914,6 +952,17 @@ class RemoteRuntime(KubeResource):
             data = json.loads(data)
         return data
 
+    def _trigger_of_kind_exists(self, kind: str) -> bool:
+        if not self.spec.config:
+            return False
+
+        for name, spec in self.spec.config.items():
+            if isinstance(spec, dict):
+                if spec.get("kind") == kind:
+                    return True
+
+        return False
+
     def _pre_run_validations(self):
         if self.spec.function_kind != "mlrun":
             raise RunError(

mlrun/runtimes/kubejob.py

@@ -175,7 +175,7 @@ class KubejobRuntime(KubeResource):
         :param show_on_failure:         show logs only in case of build failure
         :param force_build:             set True for force building the image, even when no changes were made
 
-        :return True if the function is ready (deployed)
+        :return: True if the function is ready (deployed)
         """
 
         build = self.spec.build

mlrun/runtimes/local.py

@@ -104,13 +104,13 @@ class ParallelRunner:
                 num_errors += 1
             results.append(resp)
             if num_errors > generator.max_errors:
-                logger.error("max errors reached, stopping iterations!")
+                logger.error("Max errors reached, stopping iterations!")
                 return True
             run_results = resp["status"].get("results", {})
             stop = generator.eval_stop_condition(run_results)
             if stop:
                 logger.info(
-                    f"reached early stop condition ({generator.options.stop_condition}), stopping iterations!"
+                    f"Reached early stop condition ({generator.options.stop_condition}), stopping iterations!"
                 )
             return stop
 
@@ -140,7 +140,7 @@ class ParallelRunner:
 
         client.close()
         if function_name and generator.options.teardown_dask:
-            logger.info("tearing down the dask cluster..")
+            logger.info("Tearing down the dask cluster..")
             mlrun.get_run_db().delete_runtime_resources(
                 kind="dask", object_id=function_name, force=True
             )
@@ -278,7 +278,7 @@ class LocalRuntime(BaseRuntime, ParallelRunner):
 
         handler = runobj.spec.handler
         handler_str = handler or "main"
-        logger.debug(f"starting local run: {self.spec.command} # {handler_str}")
+        logger.debug(f"Starting local run: {self.spec.command} # {handler_str}")
         pythonpath = self.spec.pythonpath
 
         if handler:
@@ -317,13 +317,13 @@ class LocalRuntime(BaseRuntime, ParallelRunner):
                 context.set_state(error=err_to_str(exc), commit=True)
                 logger.error(f"run error, {traceback.format_exc()}")
                 raise RunError(
-                    "failed on pre-loading / post-running of the function"
+                    "Failed on pre-loading / post-running of the function"
                 ) from exc
 
         else:
             command = self.spec.command
             command = command.format(**runobj.spec.parameters)
-            logger.info(f"handler was not provided running main ({command})")
+            logger.info(f"Handler was not provided running main ({command})")
             arg_list = command.split()
             if self.spec.mode == "pass":
                 cmd = arg_list
@@ -360,9 +360,9 @@ class LocalRuntime(BaseRuntime, ParallelRunner):
                 if resp:
                     run_obj_dict = json.loads(resp)
                 else:
-                    logger.error("empty context tmp file")
+                    logger.debug("Empty context tmp file")
             else:
-                logger.info("no context file found")
+                logger.info("No context file found")
 
             # If trackers where used, this is where we log all data collected to MLRun
             run_obj_dict = trackers_manager.post_run(run_obj_dict)
@@ -396,7 +396,7 @@ def run_exec(cmd, args, env=None, cwd=None):
         cmd += args
     if env and "SYSTEMROOT" in os.environ:
         env["SYSTEMROOT"] = os.environ["SYSTEMROOT"]
-    print("running:", cmd)
+    print("Running:", cmd)
     process = Popen(
         cmd, stdout=PIPE, stderr=PIPE, env=os.environ, cwd=cwd, universal_newlines=True
     )

mlrun/runtimes/pod.py

@@ -1067,7 +1067,7 @@ class KubeResource(BaseRuntime):
         Set the threshold for a specific state of the runtime.
         The threshold is the amount of time that the runtime will wait before aborting the run if the job is in the
         matching state.
-        The threshold time string must conform to timelength python package standards and be at least 1 second
+        The threshold time string must conform to timelength python package standards and be at least 1 minute
         (e.g. 1000s, 1 hour 30m, 1h etc. or -1 for infinite).
         If the threshold is not set for a state, the default threshold will be used.
 

mlrun/runtimes/remotesparkjob.py

@@ -205,6 +205,7 @@ class RemoteSparkRuntime(KubejobRuntime):
             mlrun_version_specifier=mlrun_version_specifier,
             builder_env=builder_env,
             show_on_failure=show_on_failure,
+            force_build=force_build,
         )
 
 

mlrun/runtimes/serving.py

@@ -147,6 +147,7 @@ class ServingSpec(NuclioSpec):
         add_templated_ingress_host_mode=None,
         clone_target_dir=None,
         state_thresholds=None,
+        disable_default_http_trigger=None,
     ):
         super().__init__(
             command=command,
@@ -187,6 +188,7 @@ class ServingSpec(NuclioSpec):
             service_type=service_type,
             add_templated_ingress_host_mode=add_templated_ingress_host_mode,
             clone_target_dir=clone_target_dir,
+            disable_default_http_trigger=disable_default_http_trigger,
         )
 
         self.models = models or {}
@@ -584,6 +586,7 @@ class ServingRuntime(RemoteRuntime):
         verbose=False,
         auth_info: mlrun.common.schemas.AuthInfo = None,
         builder_env: dict = None,
+        force_build: bool = False,
     ):
         """deploy model serving function to a local/remote cluster
 
@@ -594,6 +597,7 @@ class ServingRuntime(RemoteRuntime):
         :param auth_info: The auth info to use to communicate with the Nuclio dashboard, required only when providing
                           dashboard
         :param builder_env: env vars dict for source archive config/credentials e.g. builder_env={"GIT_TOKEN": token}
+        :param force_build: set True for force building the image
         """
         load_mode = self.spec.load_mode
         if load_mode and load_mode not in ["sync", "async"]:
@@ -633,7 +637,13 @@ class ServingRuntime(RemoteRuntime):
             logger.info(f"deploy root function {self.metadata.name} ...")
 
         return super().deploy(
-            dashboard, project, tag, verbose, auth_info, builder_env=builder_env
+            dashboard,
+            project,
+            tag,
+            verbose,
+            auth_info,
+            builder_env=builder_env,
+            force_build=force_build,
         )
 
     def _get_runtime_env(self):

mlrun/runtimes/sparkjob/spark3job.py

@@ -856,6 +856,7 @@ class Spark3Runtime(KubejobRuntime):
         mlrun_version_specifier=None,
         builder_env: dict = None,
         show_on_failure: bool = False,
+        force_build: bool = False,
     ):
         """deploy function, build container with dependencies
 
@@ -867,8 +868,9 @@ class Spark3Runtime(KubejobRuntime):
         :param builder_env:             Kaniko builder pod env vars dict (for config/credentials)
                                         e.g. builder_env={"GIT_TOKEN": token}
         :param show_on_failure:         show logs only in case of build failure
+        :param force_build:             set True for force building the image, even when no changes were made
 
-        :return True if the function is ready (deployed)
+        :return: True if the function is ready (deployed)
         """
         # connect will populate the config from the server config
         mlrun.db.get_run_db()
@@ -882,6 +884,7 @@ class Spark3Runtime(KubejobRuntime):
             mlrun_version_specifier=mlrun_version_specifier,
             builder_env=builder_env,
             show_on_failure=show_on_failure,
+            force_build=force_build,
         )
 
     @staticmethod