mlrun 1.9.0rc4__py3-none-any.whl → 1.9.0rc5__py3-none-any.whl

mlrun/model_monitoring/applications/evidently/base.py

@@ -14,19 +14,18 @@
 
 import json
 import posixpath
-import uuid
 import warnings
 from abc import ABC
+from tempfile import NamedTemporaryFile
+from typing import Optional
 
-import pandas as pd
 import semver
-from evidently.ui.storage.local.base import METADATA_PATH, FSLocation
 
 import mlrun.model_monitoring.applications.base as mm_base
 import mlrun.model_monitoring.applications.context as mm_context
-from mlrun.errors import MLRunIncompatibleVersionError
+from mlrun.errors import MLRunIncompatibleVersionError, MLRunValueError
 
-SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.6.0")
+SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.7.5")
 
 
 def _check_evidently_version(*, cur: semver.Version, ref: semver.Version) -> None:
@@ -60,36 +59,66 @@ except ModuleNotFoundError:
 
 
 if _HAS_EVIDENTLY:
-    from evidently.suite.base_suite import Display
-    from evidently.ui.type_aliases import STR_UUID
-    from evidently.ui.workspace import Workspace
-    from evidently.utils.dashboard import TemplateParams, file_html_template
+    from evidently.core.report import Snapshot
+    from evidently.legacy.ui.storage.local.base import METADATA_PATH, FSLocation
+    from evidently.ui.workspace import (
+        STR_UUID,
+        CloudWorkspace,
+        Project,
+        Workspace,
+        WorkspaceBase,
+    )
 
 
 class EvidentlyModelMonitoringApplicationBase(
     mm_base.ModelMonitoringApplicationBase, ABC
 ):
     def __init__(
-        self, evidently_workspace_path: str, evidently_project_id: "STR_UUID"
+        self,
+        evidently_project_id: "STR_UUID",
+        evidently_workspace_path: Optional[str] = None,
+        cloud_workspace: bool = False,
     ) -> None:
         """
-        A class for integrating Evidently for mlrun model monitoring within a monitoring application.
-        Note: evidently is not installed by default in the mlrun/mlrun image.
-        It must be installed separately to use this class.
+        A class for integrating Evidently for MLRun model monitoring within a monitoring application.
+
+        .. note::
+
+            The ``evidently`` package is not installed by default in the mlrun/mlrun image.
+            It must be installed separately to use this class.
 
-        :param evidently_workspace_path:    (str) The path to the Evidently workspace.
         :param evidently_project_id:        (str) The ID of the Evidently project.
+        :param evidently_workspace_path:    (str) The path to the Evidently workspace.
+        :param cloud_workspace:             (bool) Whether the workspace is an Evidently Cloud workspace.
         """
-
-        # TODO : more then one project (mep -> project)
         if not _HAS_EVIDENTLY:
             raise ModuleNotFoundError("Evidently is not installed - the app cannot run")
-        self._log_location(evidently_workspace_path)
-        self.evidently_workspace = Workspace.create(evidently_workspace_path)
+        self.evidently_workspace_path = evidently_workspace_path
+        if cloud_workspace:
+            self.get_workspace = self.get_cloud_workspace
+        self.evidently_workspace = self.get_workspace()
         self.evidently_project_id = evidently_project_id
-        self.evidently_project = self.evidently_workspace.get_project(
-            evidently_project_id
-        )
+        self.evidently_project = self.load_project()
+
+    def load_project(self) -> Project:
+        """Load the Evidently project."""
+        return self.evidently_workspace.get_project(self.evidently_project_id)
+
+    def get_workspace(self) -> WorkspaceBase:
+        """Get the Evidently workspace. Override this method for customize access to the workspace."""
+        if self.evidently_workspace_path:
+            self._log_location(self.evidently_workspace_path)
+            return Workspace.create(self.evidently_workspace_path)
+        else:
+            raise MLRunValueError(
+                "A local workspace could not be created as `evidently_workspace_path` is not set.\n"
+                "If you intend to use a cloud workspace, please use `cloud_workspace=True` and set the "
+                "`EVIDENTLY_API_KEY` environment variable. In other cases, override this method."
+            )
+
+    def get_cloud_workspace(self) -> CloudWorkspace:
+        """Load the Evidently cloud workspace according to the `EVIDENTLY_API_KEY` environment variable."""
+        return CloudWorkspace()
 
     @staticmethod
     def _log_location(evidently_workspace_path):
@@ -128,7 +157,7 @@ class EvidentlyModelMonitoringApplicationBase(
     @staticmethod
     def log_evidently_object(
         monitoring_context: mm_context.MonitoringApplicationContext,
-        evidently_object: "Display",
+        evidently_object: "Snapshot",
         artifact_name: str,
         unique_per_endpoint: bool = True,
     ) -> None:
@@ -141,56 +170,15 @@ class EvidentlyModelMonitoringApplicationBase(
             This method should be called on special occasions only.
 
         :param monitoring_context:  (MonitoringApplicationContext) The monitoring context to process.
-        :param evidently_object:    (Display) The Evidently display to log, e.g. a report or a test suite object.
-        :param artifact_name:       (str) The name for the logged artifact.
-        :param unique_per_endpoint: by default ``True``, we will log different artifact for each model endpoint,
-                                    set to ``False`` without changing item key will cause artifact override.
-        """
-        evidently_object_html = evidently_object.get_html()
-        monitoring_context.log_artifact(
-            artifact_name,
-            body=evidently_object_html.encode("utf-8"),
-            format="html",
-            unique_per_endpoint=unique_per_endpoint,
-        )
-
-    def log_project_dashboard(
-        self,
-        monitoring_context: mm_context.MonitoringApplicationContext,
-        timestamp_start: pd.Timestamp,
-        timestamp_end: pd.Timestamp,
-        artifact_name: str = "dashboard",
-        unique_per_endpoint: bool = True,
-    ) -> None:
-        """
-        Logs an Evidently project dashboard.
-
-        .. caution::
-
-            Logging Evidently dashboards in every model monitoring window may cause scale issues.
-            This method should be called on special occasions only.
-
-        :param monitoring_context:  (MonitoringApplicationContext) The monitoring context to process.
-        :param timestamp_start:     (pd.Timestamp) The start timestamp for the dashboard data.
-        :param timestamp_end:       (pd.Timestamp) The end timestamp for the dashboard data.
+        :param evidently_object:    (Snapshot) The Evidently run to log, e.g. a report run.
         :param artifact_name:       (str) The name for the logged artifact.
         :param unique_per_endpoint: by default ``True``, we will log different artifact for each model endpoint,
                                     set to ``False`` without changing item key will cause artifact override.
         """
-
-        dashboard_info = self.evidently_project.build_dashboard_info(
-            timestamp_start, timestamp_end
-        )
-        template_params = TemplateParams(
-            dashboard_id="pd_" + str(uuid.uuid4()).replace("-", ""),
-            dashboard_info=dashboard_info,
-            additional_graphs={},
-        )
-
-        dashboard_html = file_html_template(params=template_params)
-        monitoring_context.log_artifact(
-            artifact_name,
-            body=dashboard_html.encode("utf-8"),
-            format="html",
-            unique_per_endpoint=unique_per_endpoint,
-        )
+        with NamedTemporaryFile(suffix=".html") as file:
+            evidently_object.save_html(filename=file.name)
+            monitoring_context.log_artifact(
+                artifact_name,
+                local_path=file.name,
+                unique_per_endpoint=unique_per_endpoint,
+            )

mlrun/model_monitoring/controller.py

@@ -25,6 +25,7 @@ from types import TracebackType
 from typing import Any, NamedTuple, Optional, Union, cast
 
 import nuclio_sdk
+import pandas as pd
 
 import mlrun
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
@@ -673,9 +674,15 @@ class MonitoringApplicationController:
         """
         logger.info("Starting monitoring controller chief")
         applications_names = []
-        endpoints = self.project_obj.list_model_endpoints(
-            metric_list=["last_request"]
-        ).endpoints
+        endpoints = self.project_obj.list_model_endpoints(tsdb_metrics=False).endpoints
+        last_request_dict = self.tsdb_connector.get_last_request(
+            endpoint_ids=[mep.metadata.uid for mep in endpoints]
+        )
+        if isinstance(last_request_dict, pd.DataFrame):
+            last_request_dict = last_request_dict.set_index(
+                mm_constants.EventFieldType.ENDPOINT_ID
+            )[mm_constants.ModelEndpointSchema.LAST_REQUEST].to_dict()
+
         if not endpoints:
             logger.info("No model endpoints found", project=self.project)
             return
@@ -721,16 +728,22 @@ class MonitoringApplicationController:
             with schedules.ModelMonitoringSchedulesFileChief(
                 self.project
             ) as schedule_file:
-                futures = {
-                    pool.submit(
-                        self.endpoint_to_regular_event,
-                        endpoint,
-                        policy,
-                        set(applications_names),
-                        schedule_file,
-                    ): endpoint
-                    for endpoint in endpoints
-                }
+                for endpoint in endpoints:
+                    last_request = last_request_dict.get(endpoint.metadata.uid, None)
+                    if isinstance(last_request, float):
+                        last_request = pd.to_datetime(last_request, unit="s", utc=True)
+                    endpoint.status.last_request = (
+                        last_request or endpoint.status.last_request
+                    )
+                    futures = {
+                        pool.submit(
+                            self.endpoint_to_regular_event,
+                            endpoint,
+                            policy,
+                            set(applications_names),
+                            schedule_file,
+                        ): endpoint
+                    }
                 for future in concurrent.futures.as_completed(futures):
                     if future.exception():
                         exception = future.exception()

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -455,12 +455,20 @@ class V3IOTSDBConnector(TSDBConnector):
             # Delete all tables
             tables = mm_schemas.V3IOTSDBTables.list()
         for table_to_delete in tables:
-            try:
-                self.frames_client.delete(backend=_TSDB_BE, table=table_to_delete)
-            except v3io_frames.DeleteError as e:
+            if table_to_delete in self.tables:
+                try:
+                    self.frames_client.delete(
+                        backend=_TSDB_BE, table=self.tables[table_to_delete]
+                    )
+                except v3io_frames.DeleteError as e:
+                    logger.warning(
+                        f"Failed to delete TSDB table '{table_to_delete}'",
+                        err=mlrun.errors.err_to_str(e),
+                    )
+            else:
                 logger.warning(
-                    f"Failed to delete TSDB table '{table}'",
-                    err=mlrun.errors.err_to_str(e),
+                    f"Skipping deletion: table '{table_to_delete}' is not among the initialized tables.",
+                    initialized_tables=list(self.tables.keys()),
                 )
 
         # Final cleanup of tsdb path

mlrun/model_monitoring/tracking_policy.py

@@ -57,7 +57,7 @@ class TrackingPolicy(mlrun.model.ModelObj):
         """
         warnings.warn(
             "The `TrackingPolicy` class is deprecated from version 1.7.0 and is not "
-            "used anymore. It will be removed in 1.9.0.",
+            "used anymore. It will be removed in 1.10.0.",
             FutureWarning,
         )
 

mlrun/model_monitoring/writer.py

@@ -129,7 +129,7 @@ class ModelMonitoringWriter(StepToDict):
             )
         kind = event.pop(WriterEvent.EVENT_KIND, WriterEventKind.RESULT)
         result_event = _AppResultEvent(json.loads(event.pop(WriterEvent.DATA, "{}")))
-        if not result_event:  # BC for < 1.7.0, can be removed in 1.9.0
+        if not result_event:  # BC for < 1.7.0, can be removed in 1.10.0
             result_event = _AppResultEvent(event)
         else:
             result_event.update(_AppResultEvent(event))

mlrun/projects/operations.py

@@ -294,9 +294,9 @@ def build_function(
     :param force_build: Force building the image, even when no changes were made
     """
     if not overwrite_build_params:
-        # TODO: change overwrite_build_params default to True in 1.9.0
+        # TODO: change overwrite_build_params default to True in 1.10.0
         warnings.warn(
-            "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.9.0.",
+            "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.10.0.",
             mlrun.utils.OverwriteBuildParamsWarning,
         )
 
@@ -325,7 +325,7 @@ def build_function(
             skip_deployed=skip_deployed,
         )
     else:
-        # TODO: remove filter once overwrite_build_params default is changed to True in 1.9.0
+        # TODO: remove filter once overwrite_build_params default is changed to True in 1.10.0
         with warnings.catch_warnings():
             warnings.simplefilter(
                 "ignore", category=mlrun.utils.OverwriteBuildParamsWarning

mlrun/projects/project.py

@@ -756,10 +756,10 @@ def _project_instance_from_struct(struct, name, allow_cross_project):
         )
 
         if allow_cross_project is None:
-            # TODO: Remove this warning in version 1.9.0 and also fix cli to support allow_cross_project
+            # TODO: Remove this warning in version 1.10.0 and also fix cli to support allow_cross_project
             warnings.warn(
                 f"Project {name=} is different than specified on the context's project yaml. "
-                "This behavior is deprecated and will not be supported from version 1.9.0."
+                "This behavior is deprecated and will not be supported from version 1.10.0."
             )
             logger.warn(error_message)
         elif allow_cross_project:
@@ -2478,9 +2478,9 @@ class MlrunProject(ModelObj):
         :param fetch_credentials_from_sys_config: If true, fetch the credentials from the system configuration.
         """
         if default_controller_image != "mlrun/mlrun":
-            # TODO: Remove this in 1.9.0
+            # TODO: Remove this in 1.10.0
             warnings.warn(
-                "'default_controller_image' is deprecated and will be removed in 1.9.0, "
+                "'default_controller_image' is deprecated in 1.7.0 and will be removed in 1.10.0, "
                 "use 'image' instead",
                 FutureWarning,
             )
@@ -2860,10 +2860,10 @@ class MlrunProject(ModelObj):
 
         self.spec.set_function(name, function_object, func)
 
-    # TODO: Remove this in 1.10.0
+    # TODO: Remove this in 1.11.0
     @deprecated.deprecated(
         version="1.8.0",
-        reason="'remove_function' is deprecated and will be removed in 1.10.0. "
+        reason="'remove_function' is deprecated and will be removed in 1.11.0. "
         "Please use `delete_function` instead.",
         category=FutureWarning,
     )
@@ -2891,9 +2891,9 @@ class MlrunProject(ModelObj):
 
         :param name: name of the model-monitoring-function/s (under the project)
         """
-        # TODO: Remove this in 1.9.0
+        # TODO: Remove this in 1.10.0
         warnings.warn(
-            "'remove_model_monitoring_function' is deprecated and will be removed in 1.9.0. "
+            "'remove_model_monitoring_function' is deprecated in 1.7.0 and will be removed in 1.10.0. "
             "Please use `delete_model_monitoring_function` instead.",
             FutureWarning,
         )
@@ -3795,7 +3795,7 @@ class MlrunProject(ModelObj):
         top_level: bool = False,
         uids: Optional[list[str]] = None,
         latest_only: bool = False,
-        tsdb_metrics: bool = True,
+        tsdb_metrics: bool = False,
         metric_list: Optional[list[str]] = None,
     ) -> mlrun.common.schemas.ModelEndpointList:
         """
@@ -4064,9 +4064,9 @@ class MlrunProject(ModelObj):
             (by default `/home/mlrun_code`)
         """
         if not overwrite_build_params:
-            # TODO: change overwrite_build_params default to True in 1.9.0
+            # TODO: change overwrite_build_params default to True in 1.10.0
             warnings.warn(
-                "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.9.0.",
+                "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.10.0.",
                 mlrun.utils.OverwriteBuildParamsWarning,
             )
         default_image_name = mlrun.mlconf.default_project_image_name.format(
@@ -4141,9 +4141,9 @@ class MlrunProject(ModelObj):
             )
 
         if not overwrite_build_params:
-            # TODO: change overwrite_build_params default to True in 1.9.0
+            # TODO: change overwrite_build_params default to True in 1.10.0
             warnings.warn(
-                "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.9.0.",
+                "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.10.0.",
                 mlrun.utils.OverwriteBuildParamsWarning,
             )
 
@@ -4322,7 +4322,7 @@ class MlrunProject(ModelObj):
         :param kind: Return artifacts of the requested kind.
         :param category: Return artifacts of the requested category.
         :param tree: Return artifacts of the requested tree.
-        :param limit: Deprecated - Maximum number of artifacts to return (will be removed in 1.10.0).
+        :param limit: Deprecated - Maximum number of artifacts to return (will be removed in 1.11.0).
         :param format_: The format in which to return the artifacts. Default is 'full'.
         :param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
             parameter must be provided as well.
@@ -4335,9 +4335,9 @@ class MlrunProject(ModelObj):
         db = mlrun.db.get_run_db(secrets=self._secrets)
 
         if limit:
-            # TODO: Remove this in 1.10.0
+            # TODO: Remove this in 1.11.0
             warnings.warn(
-                "'limit' is deprecated and will be removed in 1.10.0. Use 'page' and 'page_size' instead.",
+                "'limit' is deprecated and will be removed in 1.11.0. Use 'page' and 'page_size' instead.",
                 FutureWarning,
             )
 
@@ -4467,7 +4467,7 @@ class MlrunProject(ModelObj):
             artifacts generated from a hyper-param run. If only a single iteration exists, will return the artifact
             from that iteration. If using ``best_iter``, the ``iter`` parameter must not be used.
         :param tree: Return artifacts of the requested tree.
-        :param limit: Maximum number of artifacts to return.
+        :param limit: Deprecated - Maximum number of artifacts to return (will be removed in 1.11.0).
         :param format_: The format in which to return the artifacts. Default is 'full'.
         """
         db = mlrun.db.get_run_db(secrets=self._secrets)
@@ -4734,7 +4734,7 @@ class MlrunProject(ModelObj):
         :param states: List only runs whose state is one of the provided states.
         :param sort: Whether to sort the result according to their start time. Otherwise, results will be
             returned by their internal order in the DB (order will not be guaranteed).
-        :param last: Deprecated - currently not used (will be removed in 1.9.0).
+        :param last: Deprecated - currently not used (will be removed in 1.10.0).
         :param iter: If ``True`` return runs from all iterations. Otherwise, return only runs whose ``iter`` is 0.
         :param start_time_from: Filter by run start time in ``[start_time_from, start_time_to]``.
         :param start_time_to: Filter by run start time in ``[start_time_from, start_time_to]``.
@@ -4745,9 +4745,9 @@ class MlrunProject(ModelObj):
         :param end_time_to: Filter by run end time in ``[end_time_from, end_time_to]``.
         """
         if state:
-            # TODO: Remove this in 1.9.0
+            # TODO: Remove this in 1.10.0
             warnings.warn(
-                "'state' is deprecated and will be removed in 1.9.0. Use 'states' instead.",
+                "'state' is deprecated in 1.7.0 and will be removed in 1.10.0. Use 'states' instead.",
                 FutureWarning,
             )
 

mlrun/render.py

@@ -361,9 +361,6 @@ def get_tblframe(df, display, classes=None):
     return ipython_display(html, display)
 
 
-uid_template = '<div title="{}"><a href="{}/{}/{}/jobs/monitor/{}/overview" target="_blank" >...{}</a></div>'
-
-
 def runs_to_html(
     df: pd.DataFrame,
     display: bool = True,
@@ -379,15 +376,14 @@ def runs_to_html(
     df["results"] = df["results"].apply(dict_html)
     df["start"] = df["start"].apply(time_str)
     df["parameters"] = df["parameters"].apply(dict_html)
+    uid_template = '<div title="{}"><a href="{}" target="_blank" >...{}</a></div>'
+
     if config.resolve_ui_url():
         df["uid"] = df.apply(
             lambda x: uid_template.format(
-                x.uid,
-                config.resolve_ui_url(),
-                config.ui.projects_prefix,
-                x.project,
-                x.uid,
-                x.uid[-8:],
+                x["uid"],
+                mlrun.utils.get_run_url(x["project"], x["uid"], x["name"]),
+                x["uid"][-8:],
             ),
             axis=1,
         )

mlrun/run.py

@@ -261,7 +261,7 @@ def get_or_create_ctx(
     """
     if labels:
         warnings.warn(
-            "The `labels` argument is deprecated and will be removed in 1.9.0. "
+            "The `labels` argument is deprecated in 1.7.0 and will be removed in 1.10.0. "
             "Please use `spec` instead, e.g.:\n"
             "spec={'metadata': {'labels': {'key': 'value'}}}",
             FutureWarning,

mlrun/runtimes/base.py

@@ -148,10 +148,10 @@ class FunctionSpec(ModelObj):
 
     @property
     def clone_target_dir(self):
-        # TODO: remove this property in 1.9.0
+        # TODO: remove this property in 1.10.0
         if self.build.source_code_target_dir:
             warnings.warn(
-                "The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.9.0. "
+                "The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.10.0. "
                 "Use spec.build.source_code_target_dir instead.",
                 FutureWarning,
             )
@@ -159,10 +159,10 @@ class FunctionSpec(ModelObj):
 
     @clone_target_dir.setter
     def clone_target_dir(self, clone_target_dir):
-        # TODO: remove this property in 1.9.0
+        # TODO: remove this property in 1.10.0
         if clone_target_dir:
             warnings.warn(
-                "The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.9.0. "
+                "The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.10.0. "
                 "Use spec.build.source_code_target_dir instead.",
                 FutureWarning,
             )
@@ -489,7 +489,7 @@ class BaseRuntime(ModelObj):
     def _store_function(self, runspec, meta, db):
         meta.labels["kind"] = self.kind
         mlrun.runtimes.utils.enrich_run_labels(
-            meta.labels, [mlrun.common.runtimes.constants.RunLabels.owner]
+            meta.labels, [mlrun_constants.MLRunInternalLabels.owner]
         )
         if runspec.spec.output_path:
             runspec.spec.output_path = runspec.spec.output_path.replace(

mlrun/runtimes/kubejob.py

@@ -114,9 +114,9 @@ class KubejobRuntime(KubeResource):
             e.g. builder_env={"GIT_TOKEN": token}
         """
         if not overwrite:
-            # TODO: change overwrite default to True in 1.9.0
+            # TODO: change overwrite default to True in 1.10.0
             warnings.warn(
-                "The `overwrite` parameter default will change from 'False' to 'True' in 1.9.0.",
+                "The `overwrite` parameter default will change from 'False' to 'True' in 1.10.0.",
                 mlrun.utils.OverwriteBuildParamsWarning,
             )
         image = mlrun.utils.helpers.remove_image_protocol_prefix(image)

mlrun/runtimes/nuclio/function.py

@@ -623,9 +623,9 @@ class RemoteRuntime(KubeResource):
         :param force_build: set True for force building the image
         """
         if auth_info:
-            # TODO: remove in 1.9.0
+            # TODO: remove in 1.10.0
             warnings.warn(
-                "'auth_info' is deprecated for nuclio runtimes in 1.7.0 and will be removed in 1.9.0",
+                "'auth_info' is deprecated for nuclio runtimes in 1.7.0 and will be removed in 1.10.0",
                 FutureWarning,
             )
 

mlrun/runtimes/nuclio/serving.py

@@ -44,7 +44,7 @@ from .function import NuclioSpec, RemoteRuntime, min_nuclio_versions
 serving_subkind = "serving_v2"
 
 if TYPE_CHECKING:
-    # remove this block in 1.9.0
+    # remove this block in 1.10.0
     from mlrun.model_monitoring import TrackingPolicy
 
 
@@ -361,8 +361,8 @@ class ServingRuntime(RemoteRuntime):
         if batch:
             warnings.warn(
                 "The `batch` size parameter was deprecated in version 1.8.0 and is no longer used. "
-                "It will be removed in 1.10.",
-                # TODO: Remove this in 1.10
+                "It will be removed in 1.11.",
+                # TODO: Remove this in 1.11
                 FutureWarning,
             )
         if stream_args:
@@ -370,7 +370,7 @@ class ServingRuntime(RemoteRuntime):
         if tracking_policy is not None:
             warnings.warn(
                 "The `tracking_policy` argument is deprecated from version 1.7.0 "
-                "and has no effect. It will be removed in 1.9.0.\n"
+                "and has no effect. It will be removed in 1.10.0.\n"
                 "To set the desired model monitoring time window and schedule, use "
                 "the `base_period` argument in `project.enable_model_monitoring()`.",
                 FutureWarning,

mlrun/runtimes/utils.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import enum
 import getpass
 import hashlib
 import json
@@ -28,7 +29,6 @@ import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
 import mlrun.utils.regex
 from mlrun.artifacts import TableArtifact
-from mlrun.common.runtimes.constants import RunLabels
 from mlrun.config import config
 from mlrun.errors import err_to_str
 from mlrun.frameworks.parallel_coordinates import gen_pcp_plot
@@ -433,18 +433,35 @@ def enrich_function_from_dict(function, function_dict):
 
 def enrich_run_labels(
     labels: dict,
-    labels_to_enrich: Optional[list[RunLabels]] = None,
+    labels_to_enrich: Optional[list[mlrun_constants.MLRunInternalLabels]] = None,
 ):
+    """
+    Enrich the run labels with the internal labels and the labels enrichment extension
+    :param labels: The run labels dict
+    :param labels_to_enrich: The label keys to enrich from MLRunInternalLabels.default_run_labels_to_enrich
+    :return: The enriched labels dict
+    """
+    # Merge the labels with the labels enrichment extension
     labels_enrichment = {
-        RunLabels.owner: os.environ.get("V3IO_USERNAME") or getpass.getuser(),
-        # TODO: remove this in 1.9.0
-        RunLabels.v3io_user: os.environ.get("V3IO_USERNAME"),
+        mlrun_constants.MLRunInternalLabels.owner: os.environ.get("V3IO_USERNAME")
+        or getpass.getuser(),
+        # TODO: remove this in 1.10.0
+        mlrun_constants.MLRunInternalLabels.v3io_user: os.environ.get("V3IO_USERNAME"),
     }
-    labels_to_enrich = labels_to_enrich or RunLabels.all()
+
+    # Resolve which label keys to enrich
+    if labels_to_enrich is None:
+        labels_to_enrich = (
+            mlrun_constants.MLRunInternalLabels.default_run_labels_to_enrich()
+        )
+
+    # Enrich labels
     for label in labels_to_enrich:
+        if isinstance(label, enum.Enum):
+            label = label.value
         enrichment = labels_enrichment.get(label)
-        if label.value not in labels and enrichment:
-            labels[label.value] = enrichment
+        if label not in labels and enrichment:
+            labels[label] = enrichment
     return labels
 
 

mlrun/utils/helpers.py

@@ -2224,8 +2224,9 @@ class Workflow:
             namespace=mlrun.mlconf.namespace,
         )
 
-        # arbitrary timeout of 5 seconds, the workflow should be done by now
-        kfp_run = kfp_client.wait_for_run_completion(workflow_id, 5)
+        # arbitrary timeout of 60 seconds, the workflow should be done by now, however sometimes kfp takes a few
+        # seconds to update the workflow status
+        kfp_run = kfp_client.wait_for_run_completion(workflow_id, 60)
         if not kfp_run:
             return None
 

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "f19c66499d330c4ab0f375d6fb392bf9c6ffb6db",
-  "version": "1.9.0-rc4"
+  "git_commit": "6a74fb2c76fb2f1f30c94f16336a8df6e14d5ed4",
+  "version": "1.9.0-rc5"
 }