mlrun 1.7.0rc13__py3-none-any.whl → 1.7.0rc21__py3-none-any.whl

mlrun/projects/project.py

@@ -31,23 +31,29 @@ from typing import Callable, Optional, Union
 import dotenv
 import git
 import git.exc
-import kfp
+import mlrun_pipelines.common.models
+import mlrun_pipelines.mounts
 import nuclio.utils
 import requests
 import yaml
+from mlrun_pipelines.models import PipelineNodeWrapper
 
 import mlrun.common.helpers
+import mlrun.common.runtimes.constants
+import mlrun.common.schemas.artifact
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.db
 import mlrun.errors
 import mlrun.k8s_utils
+import mlrun.model_monitoring.applications as mm_app
 import mlrun.runtimes
 import mlrun.runtimes.nuclio.api_gateway
 import mlrun.runtimes.pod
 import mlrun.runtimes.utils
 import mlrun.serving
 import mlrun.utils.regex
-from mlrun.common.schemas import AlertConfig
+from mlrun.alerts.alert import AlertConfig
+from mlrun.common.schemas.alert import AlertTemplate
 from mlrun.datastore.datastore_profile import DatastoreProfile, DatastoreProfile2Json
 from mlrun.runtimes.nuclio.function import RemoteRuntime
 
@@ -56,14 +62,10 @@ from ..artifacts.manager import ArtifactManager, dict_to_artifact, extend_artifa
 from ..datastore import store_manager
 from ..features import Feature
 from ..model import EntrypointParam, ImageBuilder, ModelObj
-from ..model_monitoring.application import (
-    ModelMonitoringApplicationBase,
-)
 from ..run import code_to_function, get_object, import_function, new_function
 from ..secrets import SecretsStore
 from ..utils import (
     is_ipython,
-    is_legacy_artifact,
     is_relative_path,
     is_yaml_path,
     logger,
@@ -76,7 +78,10 @@ from ..utils.clones import (
     clone_zip,
     get_repo_url,
 )
-from ..utils.helpers import ensure_git_branch, resolve_git_reference_from_source
+from ..utils.helpers import (
+    ensure_git_branch,
+    resolve_git_reference_from_source,
+)
 from ..utils.notifications import CustomNotificationPusher, NotificationTypes
 from .operations import (
     BuildStatus,
@@ -207,14 +212,16 @@ def new_project(
                 "Unsupported option, cannot use subpath argument with project templates"
             )
         if from_template.endswith(".yaml"):
-            project = _load_project_file(from_template, name, secrets)
+            project = _load_project_file(
+                from_template, name, secrets, allow_cross_project=True
+            )
         elif from_template.startswith("git://"):
             clone_git(from_template, context, secrets, clone=True)
             shutil.rmtree(path.join(context, ".git"))
-            project = _load_project_dir(context, name)
+            project = _load_project_dir(context, name, allow_cross_project=True)
         elif from_template.endswith(".zip"):
             clone_zip(from_template, context, secrets)
-            project = _load_project_dir(context, name)
+            project = _load_project_dir(context, name, allow_cross_project=True)
         else:
             raise ValueError("template must be a path to .yaml or .zip file")
         project.metadata.name = name
@@ -296,6 +303,7 @@ def load_project(
     save: bool = True,
     sync_functions: bool = False,
     parameters: dict = None,
+    allow_cross_project: bool = None,
 ) -> "MlrunProject":
     """Load an MLRun project from git or tar or dir
 
@@ -342,6 +350,8 @@ def load_project(
     :param save:            whether to save the created project and artifact in the DB
     :param sync_functions:  sync the project's functions into the project object (will be saved to the DB if save=True)
     :param parameters:      key/value pairs to add to the project.spec.params
+    :param allow_cross_project: if True, override the loaded project name. This flag ensures awareness of
+                                loading an existing project yaml as a baseline for a new project with a different name
 
     :returns: project object
     """
@@ -357,7 +367,7 @@ def load_project(
     if url:
         url = str(url)  # to support path objects
         if is_yaml_path(url):
-            project = _load_project_file(url, name, secrets)
+            project = _load_project_file(url, name, secrets, allow_cross_project)
             project.spec.context = context
         elif url.startswith("git://"):
             url, repo = clone_git(url, context, secrets, clone)
@@ -384,7 +394,7 @@ def load_project(
         repo, url = init_repo(context, url, init_git)
 
     if not project:
-        project = _load_project_dir(context, name, subpath)
+        project = _load_project_dir(context, name, subpath, allow_cross_project)
 
     if not project.metadata.name:
         raise ValueError("Project name must be specified")
@@ -438,6 +448,7 @@ def get_or_create_project(
     from_template: str = None,
     save: bool = True,
     parameters: dict = None,
+    allow_cross_project: bool = None,
 ) -> "MlrunProject":
     """Load a project from MLRun DB, or create/import if it does not exist
 
@@ -482,12 +493,12 @@ def get_or_create_project(
     :param from_template:     path to project YAML file that will be used as from_template (for new projects)
     :param save:         whether to save the created project in the DB
     :param parameters:   key/value pairs to add to the project.spec.params
+    :param allow_cross_project: if True, override the loaded project name. This flag ensures awareness of
+                                loading an existing project yaml as a baseline for a new project with a different name
 
     :returns: project object
     """
     context = context or "./"
-    spec_path = path.join(context, subpath or "", "project.yaml")
-    load_from_path = url or path.isfile(spec_path)
     try:
         # load project from the DB.
         # use `name` as `url` as we load the project from the DB
@@ -503,13 +514,15 @@ def get_or_create_project(
             # only loading project from db so no need to save it
             save=False,
             parameters=parameters,
+            allow_cross_project=allow_cross_project,
         )
         logger.info("Project loaded successfully", project_name=name)
         return project
-
     except mlrun.errors.MLRunNotFoundError:
         logger.debug("Project not found in db", project_name=name)
 
+    spec_path = path.join(context, subpath or "", "project.yaml")
+    load_from_path = url or path.isfile(spec_path)
     # do not nest under "try" or else the exceptions raised below will be logged along with the "not found" message
     if load_from_path:
         # loads a project from archive or local project.yaml
@@ -525,6 +538,7 @@ def get_or_create_project(
             user_project=user_project,
             save=save,
             parameters=parameters,
+            allow_cross_project=allow_cross_project,
         )
 
         logger.info(
@@ -599,7 +613,7 @@ def _run_project_setup(
     return project
 
 
-def _load_project_dir(context, name="", subpath=""):
+def _load_project_dir(context, name="", subpath="", allow_cross_project=None):
     subpath_str = subpath or ""
 
     # support both .yaml and .yml file extensions
@@ -613,7 +627,7 @@ def _load_project_dir(context, name="", subpath=""):
         with open(project_file_path) as fp:
             data = fp.read()
             struct = yaml.load(data, Loader=yaml.FullLoader)
-            project = _project_instance_from_struct(struct, name)
+            project = _project_instance_from_struct(struct, name, allow_cross_project)
             project.spec.context = context
     elif function_files := glob.glob(function_file_path):
         function_path = function_files[0]
@@ -686,19 +700,41 @@ def _delete_project_from_db(project_name, secrets, deletion_strategy):
     return db.delete_project(project_name, deletion_strategy=deletion_strategy)
 
 
-def _load_project_file(url, name="", secrets=None):
+def _load_project_file(url, name="", secrets=None, allow_cross_project=None):
     try:
         obj = get_object(url, secrets)
     except FileNotFoundError as exc:
         raise FileNotFoundError(f"cant find project file at {url}") from exc
     struct = yaml.load(obj, Loader=yaml.FullLoader)
-    return _project_instance_from_struct(struct, name)
+    return _project_instance_from_struct(struct, name, allow_cross_project)
 
 
-def _project_instance_from_struct(struct, name):
-    struct.setdefault("metadata", {})["name"] = name or struct.get("metadata", {}).get(
-        "name", ""
-    )
+def _project_instance_from_struct(struct, name, allow_cross_project):
+    name_from_struct = struct.get("metadata", {}).get("name", "")
+    if name and name_from_struct and name_from_struct != name:
+        error_message = (
+            f"project name mismatch, {name_from_struct} != {name}, please do one of the following:\n"
+            "1. Set the `allow_cross_project=True` when loading the project.\n"
+            f"2. Delete the existing project yaml, or ensure its name is equal to {name}.\n"
+            "3. Use different project context dir."
+        )
+
+        if allow_cross_project is None:
+            # TODO: Remove this warning in version 1.9.0 and also fix cli to support allow_cross_project
+            logger.warn(
+                "Project name is different than specified on its project yaml."
+                "You should fix it until version 1.9.0",
+                description=error_message,
+            )
+        elif allow_cross_project:
+            logger.warn(
+                "Project name is different than specified on its project yaml. Overriding.",
+                existing_name=name_from_struct,
+                overriding_name=name,
+            )
+        else:
+            raise ValueError(error_message)
+    struct.setdefault("metadata", {})["name"] = name or name_from_struct
     return MlrunProject.from_dict(struct)
 
 
@@ -960,13 +996,9 @@ class ProjectSpec(ModelObj):
             if not isinstance(artifact, dict) and not hasattr(artifact, "to_dict"):
                 raise ValueError("artifacts must be a dict or class")
             if isinstance(artifact, dict):
-                # Support legacy artifacts
-                if is_legacy_artifact(artifact) or _is_imported_artifact(artifact):
-                    key = artifact.get("key")
-                else:
-                    key = artifact.get("metadata").get("key", "")
+                key = artifact.get("metadata", {}).get("key", "")
                 if not key:
-                    raise ValueError('artifacts "key" must be specified')
+                    raise ValueError('artifacts "metadata.key" must be specified')
             else:
                 key = artifact.key
                 artifact = artifact.to_dict()
@@ -1243,6 +1275,14 @@ class MlrunProject(ModelObj):
     def description(self, description):
         self.spec.description = description
 
+    @property
+    def default_function_node_selector(self) -> dict:
+        return self.spec.default_function_node_selector
+
+    @default_function_node_selector.setter
+    def default_function_node_selector(self, default_function_node_selector):
+        self.spec.default_function_node_selector = default_function_node_selector
+
     @property
     def default_image(self) -> str:
         return self.spec.default_image
@@ -1562,6 +1602,23 @@ class MlrunProject(ModelObj):
         )
         return item
 
+    def delete_artifact(
+        self,
+        item: Artifact,
+        deletion_strategy: mlrun.common.schemas.artifact.ArtifactsDeletionStrategies = (
+            mlrun.common.schemas.artifact.ArtifactsDeletionStrategies.metadata_only
+        ),
+        secrets: dict = None,
+    ):
+        """Delete an artifact object in the DB and optionally delete the artifact data
+
+        :param item: Artifact object (can be any type, such as dataset, model, feature store).
+        :param deletion_strategy: The artifact deletion strategy types.
+        :param secrets: Credentials needed to access the artifact data.
+        """
+        am = self._get_artifact_manager()
+        am.delete_artifact(item, deletion_strategy, secrets)
+
     def log_dataset(
         self,
         key,
@@ -1814,10 +1871,18 @@ class MlrunProject(ModelObj):
         """
         context = context or self.spec.context
         if context:
-            project = _load_project_dir(context, self.metadata.name, self.spec.subpath)
+            project = _load_project_dir(
+                context,
+                self.metadata.name,
+                self.spec.subpath,
+                allow_cross_project=False,
+            )
         else:
             project = _load_project_file(
-                self.spec.origin_url, self.metadata.name, self._secrets
+                self.spec.origin_url,
+                self.metadata.name,
+                self._secrets,
+                allow_cross_project=None,
             )
         project.spec.source = self.spec.source
         project.spec.repo = self.spec.repo
@@ -1846,7 +1911,11 @@ class MlrunProject(ModelObj):
     def set_model_monitoring_function(
         self,
         func: typing.Union[str, mlrun.runtimes.BaseRuntime, None] = None,
-        application_class: typing.Union[str, ModelMonitoringApplicationBase] = None,
+        application_class: typing.Union[
+            str,
+            mm_app.ModelMonitoringApplicationBase,
+            mm_app.ModelMonitoringApplicationBaseV2,
+        ] = None,
         name: str = None,
         image: str = None,
         handler=None,
@@ -1862,6 +1931,7 @@ class MlrunProject(ModelObj):
         call `fn.deploy()` where `fn` is the object returned by this method.
 
         examples::
+
             project.set_model_monitoring_function(
                 name="myApp", application_class="MyApp", image="mlrun/mlrun"
             )
@@ -1884,11 +1954,6 @@ class MlrunProject(ModelObj):
                                         monitoring application's constructor.
         """
 
-        if name in mm_constants.MonitoringFunctionNames.list():
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                f"An application cannot have the following names: "
-                f"{mm_constants.MonitoringFunctionNames.list()}"
-            )
         function_object: RemoteRuntime = None
         (
             resolved_function_name,
@@ -1914,7 +1979,11 @@ class MlrunProject(ModelObj):
     def create_model_monitoring_function(
         self,
         func: str = None,
-        application_class: typing.Union[str, ModelMonitoringApplicationBase] = None,
+        application_class: typing.Union[
+            str,
+            mm_app.ModelMonitoringApplicationBase,
+            mm_app.ModelMonitoringApplicationBaseV2,
+        ] = None,
         name: str = None,
         image: str = None,
         handler: str = None,
@@ -1928,6 +1997,7 @@ class MlrunProject(ModelObj):
         Create a monitoring function object without setting it to the project
 
         examples::
+
             project.create_model_monitoring_function(
                 application_class_name="MyApp", image="mlrun/mlrun", name="myApp"
             )
@@ -1949,6 +2019,7 @@ class MlrunProject(ModelObj):
         :param application_kwargs:      Additional keyword arguments to be passed to the
                                         monitoring application's constructor.
         """
+
         _, function_object, _ = self._instantiate_model_monitoring_function(
             func,
             application_class,
@@ -1967,7 +2038,10 @@ class MlrunProject(ModelObj):
         self,
         func: typing.Union[str, mlrun.runtimes.BaseRuntime, None] = None,
         application_class: typing.Union[
-            str, ModelMonitoringApplicationBase, None
+            str,
+            mm_app.ModelMonitoringApplicationBase,
+            mm_app.ModelMonitoringApplicationBaseV2,
+            None,
         ] = None,
         name: typing.Optional[str] = None,
         image: typing.Optional[str] = None,
@@ -2024,12 +2098,24 @@ class MlrunProject(ModelObj):
 
         return resolved_function_name, function_object, func
 
+    def _wait_for_functions_deployment(self, function_names: list[str]) -> None:
+        """
+        Wait for the deployment of functions on the backend.
+
+        :param function_names: A list of function names.
+        """
+        for fn_name in function_names:
+            fn = typing.cast(RemoteRuntime, self.get_function(key=fn_name))
+            fn._wait_for_function_deployment(db=fn._get_db())
+
     def enable_model_monitoring(
         self,
         default_controller_image: str = "mlrun/mlrun",
         base_period: int = 10,
         image: str = "mlrun/mlrun",
+        *,
         deploy_histogram_data_drift_app: bool = True,
+        wait_for_deployment: bool = False,
     ) -> None:
         """
         Deploy model monitoring application controller, writer and stream functions.
@@ -2039,7 +2125,6 @@ class MlrunProject(ModelObj):
         The stream function goal is to monitor the log of the data stream. It is triggered when a new log entry
         is detected. It processes the new events into statistics that are then written to statistics databases.
 
-
         :param default_controller_image:        Deprecated.
         :param base_period:                     The time period in minutes in which the model monitoring controller
                                                 function is triggered. By default, the base period is 10 minutes.
@@ -2047,6 +2132,9 @@ class MlrunProject(ModelObj):
                                                 stream & histogram data drift functions, which are real time nuclio
                                                 functions. By default, the image is mlrun/mlrun.
         :param deploy_histogram_data_drift_app: If true, deploy the default histogram-based data drift application.
+        :param wait_for_deployment:             If true, return only after the deployment is done on the backend.
+                                                Otherwise, deploy the model monitoring infrastructure on the
+                                                background, including the histogram data drift app if selected.
         """
         if default_controller_image != "mlrun/mlrun":
             # TODO: Remove this in 1.9.0
@@ -2064,37 +2152,55 @@ class MlrunProject(ModelObj):
             deploy_histogram_data_drift_app=deploy_histogram_data_drift_app,
         )
 
+        if wait_for_deployment:
+            deployment_functions = mm_constants.MonitoringFunctionNames.list()
+            if deploy_histogram_data_drift_app:
+                deployment_functions.append(
+                    mm_constants.HistogramDataDriftApplicationConstants.NAME
+                )
+            self._wait_for_functions_deployment(deployment_functions)
+
     def deploy_histogram_data_drift_app(
         self,
         *,
         image: str = "mlrun/mlrun",
         db: Optional[mlrun.db.RunDBInterface] = None,
+        wait_for_deployment: bool = False,
     ) -> None:
         """
         Deploy the histogram data drift application.
 
-        :param image: The image on which the application will run.
-        :param db:    An optional DB object.
+        :param image:               The image on which the application will run.
+        :param db:                  An optional DB object.
+        :param wait_for_deployment: If true, return only after the deployment is done on the backend.
+                                    Otherwise, deploy the application on the background.
         """
         if db is None:
             db = mlrun.db.get_run_db(secrets=self._secrets)
         db.deploy_histogram_data_drift_app(project=self.name, image=image)
 
+        if wait_for_deployment:
+            self._wait_for_functions_deployment(
+                [mm_constants.HistogramDataDriftApplicationConstants.NAME]
+            )
+
     def update_model_monitoring_controller(
         self,
         base_period: int = 10,
         image: str = "mlrun/mlrun",
+        *,
+        wait_for_deployment: bool = False,
     ) -> None:
         """
         Redeploy model monitoring application controller functions.
 
-
-        :param base_period:              The time period in minutes in which the model monitoring controller function
-                                         is triggered. By default, the base period is 10 minutes.
-        :param image:                    The image of the model monitoring controller, writer & monitoring
-                                         stream functions, which are real time nuclio functions.
-                                         By default, the image is mlrun/mlrun.
-        :returns: model monitoring controller job as a dictionary.
+        :param base_period:         The time period in minutes in which the model monitoring controller function
+                                    is triggered. By default, the base period is 10 minutes.
+        :param image:               The image of the model monitoring controller, writer & monitoring
+                                    stream functions, which are real time nuclio functions.
+                                    By default, the image is mlrun/mlrun.
+        :param wait_for_deployment: If true, return only after the deployment is done on the backend.
+                                    Otherwise, deploy the controller on the background.
         """
         db = mlrun.db.get_run_db(secrets=self._secrets)
         db.update_model_monitoring_controller(
@@ -2103,24 +2209,73 @@ class MlrunProject(ModelObj):
             image=image,
         )
 
+        if wait_for_deployment:
+            self._wait_for_functions_deployment(
+                [mm_constants.MonitoringFunctionNames.APPLICATION_CONTROLLER]
+            )
+
     def disable_model_monitoring(
-        self, *, delete_histogram_data_drift_app: bool = True
+        self,
+        *,
+        delete_resources: bool = True,
+        delete_stream_function: bool = False,
+        delete_histogram_data_drift_app: bool = True,
+        delete_user_applications: bool = False,
+        user_application_list: list[str] = None,
     ) -> None:
         """
-        Note: This method is currently not advised for use. See ML-3432.
-        Disable model monitoring by deleting the underlying functions infrastructure from MLRun database.
-
-        :param delete_histogram_data_drift_app: Whether to delete the histogram data drift app.
+        Disable model monitoring application controller, writer, stream, histogram data drift application
+        and the user's applications functions, according to the given params.
+
+        :param delete_resources:                    If True, it would delete the model monitoring controller & writer
+                                                    functions. Default True
+        :param delete_stream_function:              If True, it would delete model monitoring stream function,
+                                                    need to use wisely because if you're deleting this function
+                                                    this can cause data loss in case you will want to
+                                                    enable the model monitoring capability to the project.
+                                                    Default False.
+        :param delete_histogram_data_drift_app:     If True, it would delete the default histogram-based data drift
+                                                    application. Default False.
+        :param delete_user_applications:            If True, it would delete the user's model monitoring
+                                                    application according to user_application_list, Default False.
+        :param user_application_list:               List of the user's model monitoring application to disable.
+                                                    Default all the applications.
+                                                    Note: you have to set delete_user_applications to True
+                                                    in order to delete the desired application.
         """
-        db = mlrun.db.get_run_db(secrets=self._secrets)
-        for fn_name in mm_constants.MonitoringFunctionNames.list():
-            db.delete_function(project=self.name, name=fn_name)
-        if delete_histogram_data_drift_app:
-            db.delete_function(
-                project=self.name,
-                name=mm_constants.HistogramDataDriftApplicationConstants.NAME,
+        if not delete_user_applications and user_application_list:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "user_application_list can be specified only if delete_user_applications is set to True"
             )
 
+        db = mlrun.db.get_run_db(secrets=self._secrets)
+        succeed = db.disable_model_monitoring(
+            project=self.name,
+            delete_resources=delete_resources,
+            delete_stream_function=delete_stream_function,
+            delete_histogram_data_drift_app=delete_histogram_data_drift_app,
+            delete_user_applications=delete_user_applications,
+            user_application_list=user_application_list,
+        )
+        if succeed and delete_resources:
+            if delete_resources:
+                logger.info("Model Monitoring disabled", project=self.name)
+            if delete_user_applications:
+                logger.info(
+                    "All the desired monitoring application were deleted",
+                    project=self.name,
+                )
+        else:
+            if delete_resources:
+                logger.info(
+                    "Model Monitoring was not disabled properly", project=self.name
+                )
+            if delete_user_applications:
+                logger.info(
+                    "Some of the desired monitoring application were not deleted",
+                    project=self.name,
+                )
+
     def set_function(
         self,
         func: typing.Union[str, mlrun.runtimes.BaseRuntime] = None,
@@ -2315,22 +2470,47 @@ class MlrunProject(ModelObj):
         """
         self.spec.remove_function(name)
 
-    def remove_model_monitoring_function(self, name):
-        """remove the specified model-monitoring-app function from the project and from the db
+    def remove_model_monitoring_function(self, name: Union[str, list[str]]):
+        """remove the specified model-monitoring-app function/s from the project spec
 
-        :param name: name of the model-monitoring-app function (under the project)
+        :param name: name of the model-monitoring-function/s (under the project)
         """
-        function = self.get_function(key=name)
-        if (
-            function.metadata.labels.get(mm_constants.ModelMonitoringAppLabel.KEY)
-            == mm_constants.ModelMonitoringAppLabel.VAL
-        ):
-            self.remove_function(name=name)
-            mlrun.db.get_run_db().delete_function(name=name.lower())
-            logger.info(f"{name} function has been removed from {self.name} project")
+        names = name if isinstance(name, list) else [name]
+        for func_name in names:
+            function = self.get_function(key=func_name)
+            if (
+                function.metadata.labels.get(mm_constants.ModelMonitoringAppLabel.KEY)
+                == mm_constants.ModelMonitoringAppLabel.VAL
+            ):
+                self.remove_function(name=func_name)
+                logger.info(
+                    f"{func_name} function has been removed from {self.name} project"
+                )
+            else:
+                raise logger.warn(
+                    f"There is no model monitoring function with {func_name} name"
+                )
+
+    def delete_model_monitoring_function(self, name: Union[str, list[str]]):
+        """delete the specified model-monitoring-app function/s
+
+        :param name: name of the model-monitoring-function/s (under the project)
+        """
+        db = mlrun.db.get_run_db(secrets=self._secrets)
+        succeed = db.delete_model_monitoring_function(
+            project=self.name,
+            functions=name if isinstance(name, list) else [name],
+        )
+        if succeed:
+            logger.info(
+                "All the desired monitoring functions were deleted",
+                project=self.name,
+                functions=name,
+            )
         else:
-            raise logger.error(
-                f"There is no model monitoring function with {name} name"
+            logger.info(
+                "Some of the desired monitoring functions were not deleted",
+                project=self.name,
             )
 
     def get_function(
@@ -2438,10 +2618,10 @@ class MlrunProject(ModelObj):
     def create_remote(self, url, name="origin", branch=None):
         """Create remote for the project git
 
-         This method creates a new remote repository associated with the project's Git repository.
-         If a remote with the specified name already exists, it will not be overwritten.
+        This method creates a new remote repository associated with the project's Git repository.
+        If a remote with the specified name already exists, it will not be overwritten.
 
-         If you wish to update the URL of an existing remote, use the `set_remote` method instead.
+        If you wish to update the URL of an existing remote, use the `set_remote` method instead.
 
         :param url:    remote git url
         :param name:   name for the remote (default is 'origin')
@@ -2793,21 +2973,23 @@ class MlrunProject(ModelObj):
                           (which will be converted to the class using its `from_crontab` constructor),
                           see this link for help:
                           https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron
-                          for using the pre-defined workflow's schedule, set `schedule=True`
+                          For using the pre-defined workflow's schedule, set `schedule=True`
         :param timeout:   Timeout in seconds to wait for pipeline completion (watch will be activated)
         :param source:    Source to use instead of the actual `project.spec.source` (used when engine is remote).
-                          Can be a one of:
-                            1. Remote URL which is loaded dynamically to the workflow runner.
-                            2. A path to the project's context on the workflow runner's image.
+                          Can be one of:
+
+                          * Remote URL which is loaded dynamically to the workflow runner.
+                          * A path to the project's context on the workflow runner's image.
                           Path can be absolute or relative to `project.spec.build.source_code_target_dir` if defined
                           (enriched when building a project image with source, see `MlrunProject.build_image`).
                           For other engines the source is used to validate that the code is up-to-date.
         :param cleanup_ttl:
                           Pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
-                          Workflow and all its resources are deleted)
+                          workflow and all its resources are deleted)
         :param notifications:
                           List of notifications to send for workflow completion
-        :returns: Run id
+
+        :returns: ~py:class:`~mlrun.projects.pipelines._PipelineRunStatus` instance
         """
 
         arguments = arguments or {}
@@ -2824,12 +3006,14 @@ class MlrunProject(ModelObj):
                 "Remote repo is not defined, use .create_remote() + push()"
             )
 
-        self.sync_functions(always=sync)
-        if not self.spec._function_objects:
-            raise ValueError(
-                "There are no functions in the project."
-                " Make sure you've set your functions with project.set_function()."
-            )
+        if engine not in ["remote"]:
+            # for remote runs we don't require the functions to be synced as they can be loaded dynamically during run
+            self.sync_functions(always=sync)
+            if not self.spec._function_objects:
+                raise ValueError(
+                    "There are no functions in the project."
+                    " Make sure you've set your functions with project.set_function()."
+                )
 
         if not name and not workflow_path and not workflow_handler:
             raise ValueError("Workflow name, path, or handler must be specified")
@@ -2863,8 +3047,12 @@ class MlrunProject(ModelObj):
             engine = "remote"
         # The default engine is kfp if not given:
         workflow_engine = get_workflow_engine(engine or workflow_spec.engine, local)
-        if not inner_engine and engine == "remote":
-            inner_engine = get_workflow_engine(workflow_spec.engine, local).engine
+        if not inner_engine and workflow_engine.engine == "remote":
+            # if inner engine is set to remote, assume kfp as the default inner engine with remote as the runner
+            engine_kind = (
+                workflow_spec.engine if workflow_spec.engine != "remote" else "kfp"
+            )
+            inner_engine = get_workflow_engine(engine_kind, local).engine
         workflow_spec.engine = inner_engine or workflow_engine.engine
 
         run = workflow_engine.run(
@@ -2879,7 +3067,7 @@ class MlrunProject(ModelObj):
             notifications=notifications,
         )
         # run is None when scheduling
-        if run and run.state == mlrun.run.RunStatuses.failed:
+        if run and run.state == mlrun_pipelines.common.models.RunStatuses.failed:
             return run
         if not workflow_spec.schedule:
             # Failure and schedule messages already logged
@@ -2888,14 +3076,17 @@ class MlrunProject(ModelObj):
             )
         workflow_spec.clear_tmp()
         if (timeout or watch) and not workflow_spec.schedule:
+            run_status_kwargs = {}
             status_engine = run._engine
             # run's engine gets replaced with inner engine if engine is remote,
             # so in that case we need to get the status from the remote engine manually
-            # TODO: support watch for remote:local
-            if engine == "remote" and status_engine.engine != "local":
+            if workflow_engine.engine == "remote":
                 status_engine = _RemoteRunner
+                run_status_kwargs["inner_engine"] = run._engine
 
-            status_engine.get_run_status(project=self, run=run, timeout=timeout)
+            status_engine.get_run_status(
+                project=self, run=run, timeout=timeout, **run_status_kwargs
+            )
         return run
 
     def save_workflow(self, name, target, artifact_path=None, ttl=None):
@@ -2995,17 +3186,18 @@ class MlrunProject(ModelObj):
 
     def set_model_monitoring_credentials(
         self,
-        access_key: str = None,
-        endpoint_store_connection: str = None,
-        stream_path: str = None,
+        access_key: Optional[str] = None,
+        endpoint_store_connection: Optional[str] = None,
+        stream_path: Optional[str] = None,
+        tsdb_connection: Optional[str] = None,
     ):
         """Set the credentials that will be used by the project's model monitoring
         infrastructure functions.
 
-        :param access_key:                Model Monitoring access key for managing user permissions
         :param access_key:                Model Monitoring access key for managing user permissions
         :param endpoint_store_connection: Endpoint store connection string
         :param stream_path:               Path to the model monitoring stream
+        :param tsdb_connection:           Connection string to the time series database
         """
 
         secrets_dict = {}
@@ -3028,6 +3220,16 @@ class MlrunProject(ModelObj):
                 mlrun.common.schemas.model_monitoring.ProjectSecretKeys.STREAM_PATH
             ] = stream_path
 
+        if tsdb_connection:
+            if not tsdb_connection.startswith("taosws://"):
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "Currently only TDEngine websocket connection is supported for non-v3io TSDB,"
+                    "please provide a full URL (e.g. taosws://user:password@host:port)"
+                )
+            secrets_dict[
+                mlrun.common.schemas.model_monitoring.ProjectSecretKeys.TSDB_CONNECTION
+            ] = tsdb_connection
+
         self.set_secrets(
             secrets=secrets_dict,
             provider=mlrun.common.schemas.SecretProviderName.kubernetes,
@@ -3056,7 +3258,7 @@ class MlrunProject(ModelObj):
         notifications: list[mlrun.model.Notification] = None,
         returns: Optional[list[Union[str, dict[str, str]]]] = None,
         builder_env: Optional[dict] = None,
-    ) -> typing.Union[mlrun.model.RunObject, kfp.dsl.ContainerOp]:
+    ) -> typing.Union[mlrun.model.RunObject, PipelineNodeWrapper]:
         """Run a local or remote task as part of a local/kubeflow pipeline
 
         example (use with project)::
@@ -3112,7 +3314,7 @@ class MlrunProject(ModelObj):
                                   artifact type can be given there. The artifact key must appear in the dictionary as
                                   "key": "the_key".
         :param builder_env: env vars dict for source archive config/credentials e.g. builder_env={"GIT_TOKEN": token}
-        :return: MLRun RunObject or KubeFlow containerOp
+        :return: MLRun RunObject or PipelineNodeWrapper
         """
         return run_function(
             function,
@@ -3155,7 +3357,7 @@ class MlrunProject(ModelObj):
         requirements_file: str = None,
         extra_args: str = None,
         force_build: bool = False,
-    ) -> typing.Union[BuildStatus, kfp.dsl.ContainerOp]:
+    ) -> typing.Union[BuildStatus, PipelineNodeWrapper]:
         """deploy ML function, build container with its dependencies
 
         :param function:            name of the function (in the project) or function object
@@ -3266,7 +3468,6 @@ class MlrunProject(ModelObj):
         image: str = None,
         set_as_default: bool = True,
         with_mlrun: bool = None,
-        skip_deployed: bool = False,
         base_image: str = None,
         commands: list = None,
         secret_name: str = None,
@@ -3277,7 +3478,7 @@ class MlrunProject(ModelObj):
         requirements_file: str = None,
         extra_args: str = None,
         target_dir: str = None,
-    ) -> typing.Union[BuildStatus, kfp.dsl.ContainerOp]:
+    ) -> typing.Union[BuildStatus, PipelineNodeWrapper]:
         """Builder docker image for the project, based on the project's build config. Parameters allow to override
         the build config.
         If the project has a source configured and pull_at_runtime is not configured, this source will be cloned to the
@@ -3287,7 +3488,6 @@ class MlrunProject(ModelObj):
                         used. If not set, the `mlconf.default_project_image_name` value will be used
         :param set_as_default: set `image` to be the project's default image (default False)
         :param with_mlrun:      add the current mlrun package to the container build
-        :param skip_deployed:   *Deprecated* parameter is ignored
         :param base_image:      base image name/path (commands and source code will be added to it) defaults to
                                 mlrun.mlconf.default_base_image
         :param commands:        list of docker build (RUN) commands e.g. ['pip install pandas']
@@ -3312,14 +3512,6 @@ class MlrunProject(ModelObj):
                 base_image=base_image,
             )
 
-        if skip_deployed:
-            warnings.warn(
-                "The 'skip_deployed' parameter is deprecated and will be removed in 1.7.0. "
-                "This parameter is ignored.",
-                # TODO: remove in 1.7.0
-                FutureWarning,
-            )
-
         if not overwrite_build_params:
             # TODO: change overwrite_build_params default to True in 1.8.0
             warnings.warn(
@@ -3397,7 +3589,7 @@ class MlrunProject(ModelObj):
         verbose: bool = None,
         builder_env: dict = None,
         mock: bool = None,
-    ) -> typing.Union[DeployStatus, kfp.dsl.ContainerOp]:
+    ) -> typing.Union[DeployStatus, PipelineNodeWrapper]:
         """deploy real-time (nuclio based) functions
 
         :param function:    name of the function (in the project) or function object
@@ -3582,9 +3774,7 @@ class MlrunProject(ModelObj):
         :returns: List of function objects.
         """
 
-        model_monitoring_labels_list = [
-            f"{mm_constants.ModelMonitoringAppLabel.KEY}={mm_constants.ModelMonitoringAppLabel.VAL}"
-        ]
+        model_monitoring_labels_list = [str(mm_constants.ModelMonitoringAppLabel())]
         if labels:
             model_monitoring_labels_list += labels
         return self.list_functions(
@@ -3598,7 +3788,10 @@ class MlrunProject(ModelObj):
         name: Optional[str] = None,
         uid: Optional[Union[str, list[str]]] = None,
         labels: Optional[Union[str, list[str]]] = None,
-        state: Optional[str] = None,
+        state: Optional[
+            mlrun.common.runtimes.constants.RunStates
+        ] = None,  # Backward compatibility
+        states: typing.Optional[list[mlrun.common.runtimes.constants.RunStates]] = None,
         sort: bool = True,
         last: int = 0,
         iter: bool = False,
@@ -3632,10 +3825,11 @@ class MlrunProject(ModelObj):
         :param labels:  A list of labels to filter by. Label filters work by either filtering a specific value
                 of a label (i.e. list("key=value")) or by looking for the existence of a given
                 key (i.e. "key").
-        :param state: List only runs whose state is specified.
+        :param state: Deprecated - List only runs whose state is specified.
+        :param states: List only runs whose state is one of the provided states.
         :param sort: Whether to sort the result according to their start time. Otherwise, results will be
             returned by their internal order in the DB (order will not be guaranteed).
-        :param last: Deprecated - currently not used (will be removed in 1.8.0).
+        :param last: Deprecated - currently not used (will be removed in 1.9.0).
         :param iter: If ``True`` return runs from all iterations. Otherwise, return only runs whose ``iter`` is 0.
         :param start_time_from: Filter by run start time in ``[start_time_from, start_time_to]``.
         :param start_time_to: Filter by run start time in ``[start_time_from, start_time_to]``.
@@ -3643,13 +3837,22 @@ class MlrunProject(ModelObj):
             last_update_time_to)``.
         :param last_update_time_to: Filter by run last update time in ``(last_update_time_from, last_update_time_to)``.
         """
+        if state:
+            # TODO: Remove this in 1.9.0
+            warnings.warn(
+                "'state' is deprecated and will be removed in 1.9.0. Use 'states' instead.",
+                FutureWarning,
+            )
+
         db = mlrun.db.get_run_db(secrets=self._secrets)
         return db.list_runs(
             name,
             uid,
             self.metadata.name,
             labels=labels,
-            state=state,
+            states=mlrun.utils.helpers.as_list(state)
+            if state is not None
+            else states or None,
             sort=sort,
             last=last,
             iter=iter,
@@ -3744,15 +3947,15 @@ class MlrunProject(ModelObj):
         on MLRun and Nuclio sides, such as the 'host' attribute.
         Nuclio docs here: https://docs.nuclio.io/en/latest/reference/api-gateway/http.html
 
-        :param api_gateway: An instance of :py:class:`~mlrun.runtimes.nuclio.APIGateway` representing the configuration
-        of the API Gateway to be created or updated.
-        :param wait_for_readiness: (Optional) A boolean indicating whether to wait for the API Gateway to become ready
-            after creation or update (default is True)
-        :param max_wait_time: (Optional) Maximum time to wait for API Gateway readiness in seconds (default is 90s)
+        :param api_gateway:        An instance of :py:class:`~mlrun.runtimes.nuclio.APIGateway` representing the
+            configuration of the API Gateway to be created or updated.
+        :param wait_for_readiness: (Optional) A boolean indicating whether to wait for the API Gateway to become
+            ready after creation or update (default is True).
+        :param max_wait_time:      (Optional) Maximum time to wait for API Gateway readiness in seconds (default is 90s)
 
 
-        @return: An instance of :py:class:`~mlrun.runtimes.nuclio.APIGateway` with all fields populated based on the
-        information retrieved from the Nuclio API
+        :returns: An instance of :py:class:`~mlrun.runtimes.nuclio.APIGateway` with all fields populated based on the
+            information retrieved from the Nuclio API
         """
 
         api_gateway_json = mlrun.db.get_run_db().store_api_gateway(
@@ -3774,8 +3977,8 @@ class MlrunProject(ModelObj):
         """
         Retrieves a list of Nuclio API gateways associated with the project.
 
-        @return: List of :py:class:`~mlrun.runtimes.nuclio.api_gateway.APIGateway` objects representing
-        the Nuclio API gateways associated with the project.
+        :returns: List of :py:class:`~mlrun.runtimes.nuclio.api_gateway.APIGateway` objects representing
+           the Nuclio API gateways associated with the project.
         """
         gateways_list = mlrun.db.get_run_db().list_api_gateways(self.name)
         return [
@@ -3811,9 +4014,12 @@ class MlrunProject(ModelObj):
 
         mlrun.db.get_run_db().delete_api_gateway(name=name, project=self.name)
 
-    def store_alert_config(self, alert_data: AlertConfig, alert_name=None):
+    def store_alert_config(
+        self, alert_data: AlertConfig, alert_name=None
+    ) -> AlertConfig:
         """
         Create/modify an alert.
+
         :param alert_data: The data of the alert.
         :param alert_name: The name of the alert.
         :return: the created/modified alert.
@@ -3821,20 +4027,22 @@ class MlrunProject(ModelObj):
         db = mlrun.db.get_run_db(secrets=self._secrets)
         if alert_name is None:
             alert_name = alert_data.name
-        return db.store_alert_config(alert_name, alert_data.dict(), self.metadata.name)
+        return db.store_alert_config(alert_name, alert_data, project=self.metadata.name)
 
     def get_alert_config(self, alert_name: str) -> AlertConfig:
         """
         Retrieve an alert.
+
         :param alert_name: The name of the alert to retrieve.
         :return: The alert object.
         """
         db = mlrun.db.get_run_db(secrets=self._secrets)
         return db.get_alert_config(alert_name, self.metadata.name)
 
-    def list_alerts_configs(self):
+    def list_alerts_configs(self) -> list[AlertConfig]:
         """
         Retrieve list of alerts of a project.
+
         :return: All the alerts objects of the project.
         """
         db = mlrun.db.get_run_db(secrets=self._secrets)
@@ -3845,6 +4053,7 @@ class MlrunProject(ModelObj):
     ):
         """
         Delete an alert.
+
         :param alert_data: The data of the alert.
         :param alert_name: The name of the alert to delete.
         """
@@ -3864,6 +4073,7 @@ class MlrunProject(ModelObj):
     ):
         """
         Reset an alert.
+
         :param alert_data: The data of the alert.
         :param alert_name: The name of the alert to reset.
         """
@@ -3878,12 +4088,31 @@ class MlrunProject(ModelObj):
             alert_name = alert_data.name
         db.reset_alert_config(alert_name, self.metadata.name)
 
+    def get_alert_template(self, template_name: str) -> AlertTemplate:
+        """
+        Retrieve a specific alert template.
+
+        :param template_name: The name of the template to retrieve.
+        :return: The template object.
+        """
+        db = mlrun.db.get_run_db(secrets=self._secrets)
+        return db.get_alert_template(template_name)
+
+    def list_alert_templates(self) -> list[AlertTemplate]:
+        """
+        Retrieve list of all alert templates.
+
+        :return: All the alert template objects in the database.
+        """
+        db = mlrun.db.get_run_db(secrets=self._secrets)
+        return db.list_alert_templates()
+
     def _run_authenticated_git_action(
         self,
         action: Callable,
         remote: str,
-        args: list = [],
-        kwargs: dict = {},
+        args: list = None,
+        kwargs: dict = None,
         secrets: Union[SecretsStore, dict] = None,
     ):
         """Run an arbitrary Git routine while the remote is enriched with secrets
@@ -3903,6 +4132,8 @@ class MlrunProject(ModelObj):
         try:
             if is_remote_enriched:
                 self.spec.repo.remotes[remote].set_url(enriched_remote, clean_remote)
+            args = args or []
+            kwargs = kwargs or {}
             action(*args, **kwargs)
         except RuntimeError as e:
             raise mlrun.errors.MLRunRuntimeError(
@@ -3978,12 +4209,21 @@ class MlrunProject(ModelObj):
             else:
                 producer_dict = artifact.spec.producer
 
+            producer_tag = producer_dict.get("tag", None)
+            producer_project = producer_dict.get("project", None)
+            if not producer_tag or not producer_project:
+                # try resolving the producer tag from the uri
+                producer_uri = artifact.spec.producer.get("uri", "")
+                producer_project, producer_tag, _ = ArtifactProducer.parse_uri(
+                    producer_uri
+                )
+
             if producer_dict.get("kind", "") == "run":
                 return ArtifactProducer(
                     name=producer_dict.get("name", ""),
                     kind=producer_dict.get("kind", ""),
-                    project=producer_dict.get("project", ""),
-                    tag=producer_dict.get("tag", ""),
+                    project=producer_project,
+                    tag=producer_tag,
                 ), True
 
         # do not retain the artifact's producer, replace it with the project as the producer