mlrun 1.7.0rc5__py3-none-any.whl → 1.7.0rc7__py3-none-any.whl

mlrun/model_monitoring/{stores/kv_model_endpoint_store.py → db/stores/v3io_kv/kv_store.py}

@@ -16,17 +16,18 @@
 import json
 import os
 import typing
+from http import HTTPStatus
 
 import v3io.dataplane
+import v3io.dataplane.response
 import v3io_frames
 
 import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas.model_monitoring
+import mlrun.model_monitoring.db
 import mlrun.utils.v3io_clients
 from mlrun.utils import logger
 
-from .model_endpoint_store import ModelEndpointStore
-
 # Fields to encode before storing in the KV table or to decode after retrieving
 fields_to_encode_decode = [
     mlrun.common.schemas.model_monitoring.EventFieldType.FEATURE_STATS,
@@ -34,7 +35,7 @@ fields_to_encode_decode = [
 ]
 
 
-class KVModelEndpointStore(ModelEndpointStore):
+class KVStoreBase(mlrun.model_monitoring.db.StoreBase):
     """
     Handles the DB operations when the DB target is from type KV. For the KV operations, we use an instance of V3IO
     client and usually the KV table can be found under v3io:///users/pipelines/project-name/model-endpoints/endpoints/.
@@ -394,6 +395,128 @@ class KVModelEndpointStore(ModelEndpointStore):
 
         return metrics_mapping
 
+    def write_application_result(self, event: dict[str, typing.Any]):
+        """
+        Write a new application result event in the target table.
+
+        :param event: An event dictionary that represents the application result, should be corresponded to the
+                      schema defined in the :py:class:`~mlrun.common.schemas.model_monitoring.constants.WriterEvent`
+                      object.
+        """
+        endpoint_id = event.pop(
+            mlrun.common.schemas.model_monitoring.WriterEvent.ENDPOINT_ID
+        )
+        app_name = event.pop(
+            mlrun.common.schemas.model_monitoring.WriterEvent.APPLICATION_NAME
+        )
+        metric_name = event.pop(
+            mlrun.common.schemas.model_monitoring.WriterEvent.RESULT_NAME
+        )
+        attributes = {metric_name: json.dumps(event)}
+
+        v3io_monitoring_apps_container = self.get_v3io_monitoring_apps_container(
+            project_name=self.project
+        )
+
+        self.client.kv.update(
+            container=v3io_monitoring_apps_container,
+            table_path=endpoint_id,
+            key=app_name,
+            attributes=attributes,
+        )
+
+        schema_file = self.client.kv.new_cursor(
+            container=v3io_monitoring_apps_container,
+            table_path=endpoint_id,
+            filter_expression='__name==".#schema"',
+        )
+
+        if not schema_file.all():
+            logger.info(
+                "Generate a new V3IO KV schema file",
+                container=v3io_monitoring_apps_container,
+                endpoint_id=endpoint_id,
+            )
+            self._generate_kv_schema(endpoint_id, v3io_monitoring_apps_container)
+        logger.info("Updated V3IO KV successfully", key=app_name)
+
+    def _generate_kv_schema(
+        self, endpoint_id: str, v3io_monitoring_apps_container: str
+    ):
+        """Generate V3IO KV schema file which will be used by the model monitoring applications dashboard in Grafana."""
+        fields = [
+            {
+                "name": mlrun.common.schemas.model_monitoring.WriterEvent.RESULT_NAME,
+                "type": "string",
+                "nullable": False,
+            }
+        ]
+        res = self.client.kv.create_schema(
+            container=v3io_monitoring_apps_container,
+            table_path=endpoint_id,
+            key=mlrun.common.schemas.model_monitoring.WriterEvent.APPLICATION_NAME,
+            fields=fields,
+        )
+        if res.status_code != HTTPStatus.OK:
+            raise mlrun.errors.MLRunBadRequestError(
+                f"Couldn't infer schema for endpoint {endpoint_id} which is required for Grafana dashboards"
+            )
+        else:
+            logger.info(
+                "Generated V3IO KV schema successfully", endpoint_id=endpoint_id
+            )
+
+    def get_last_analyzed(self, endpoint_id: str, application_name: str) -> int:
+        """
+        Get the last analyzed time for the provided model endpoint and application.
+
+        :param endpoint_id:      The unique id of the model endpoint.
+        :param application_name: Registered application name.
+
+        :return: Timestamp as a Unix time.
+        :raise:  MLRunNotFoundError if last analyzed value is not found.
+
+        """
+        try:
+            data = self.client.kv.get(
+                container=self._get_monitoring_schedules_container(
+                    project_name=self.project
+                ),
+                table_path=endpoint_id,
+                key=application_name,
+            )
+            return data.output.item[
+                mlrun.common.schemas.model_monitoring.SchedulingKeys.LAST_ANALYZED
+            ]
+        except v3io.dataplane.response.HttpResponseError as err:
+            logger.debug("Error while getting last analyzed time", err=err)
+            raise mlrun.errors.MLRunNotFoundError(
+                f"No last analyzed value has been found for {application_name} "
+                f"that processes model endpoint {endpoint_id}",
+            )
+
+    def update_last_analyzed(
+        self, endpoint_id: str, application_name: str, last_analyzed: int
+    ):
+        """
+        Update the last analyzed time for the provided model endpoint and application.
+
+        :param endpoint_id:      The unique id of the model endpoint.
+        :param application_name: Registered application name.
+        :param last_analyzed:    Timestamp as a Unix time that represents the last analyzed time of a certain
+                                 application and model endpoint.
+        """
+        self.client.kv.put(
+            container=self._get_monitoring_schedules_container(
+                project_name=self.project
+            ),
+            table_path=endpoint_id,
+            key=application_name,
+            attributes={
+                mlrun.common.schemas.model_monitoring.SchedulingKeys.LAST_ANALYZED: last_analyzed
+            },
+        )
+
     def _generate_tsdb_paths(self) -> tuple[str, str]:
         """Generate a short path to the TSDB resources and a filtered path for the frames object
         :return: A tuple of:
@@ -572,3 +695,11 @@ class KVModelEndpointStore(ModelEndpointStore):
         if isinstance(field, bytes):
             return field.decode()
         return field
+
+    @staticmethod
+    def get_v3io_monitoring_apps_container(project_name: str) -> str:
+        return f"users/pipelines/{project_name}/monitoring-apps"
+
+    @staticmethod
+    def _get_monitoring_schedules_container(project_name: str) -> str:
+        return f"users/pipelines/{project_name}/monitoring-schedules/functions"

mlrun/model_monitoring/features_drift_table.py

@@ -21,9 +21,34 @@ import plotly.graph_objects as go
 from plotly.subplots import make_subplots
 
 import mlrun.common.schemas.model_monitoring
+from mlrun.artifacts import PlotlyArtifact
 
 # A type for representing a drift result, a tuple of the status and the drift mean:
-DriftResultType = tuple[mlrun.common.schemas.model_monitoring.DriftStatus, float]
+DriftResultType = tuple[
+    mlrun.common.schemas.model_monitoring.constants.ResultStatusApp, float
+]
+
+
+class _PlotlyTableArtifact(PlotlyArtifact):
+    """A custom class for plotly table artifacts"""
+
+    @staticmethod
+    def _disable_table_dragging(figure_html: str) -> str:
+        """
+        Disable the table columns dragging by adding the following
+        JavaScript code
+        """
+        start, end = figure_html.rsplit(";", 1)
+        middle = (
+            ';for (const element of document.getElementsByClassName("table")) '
+            '{element.style.pointerEvents = "none";}'
+        )
+        figure_html = start + middle + end
+        return figure_html
+
+    def get_body(self) -> str:
+        """Get the adjusted HTML representation of the figure"""
+        return self._disable_table_dragging(super().get_body())
 
 
 class FeaturesDriftTablePlot:
@@ -62,9 +87,9 @@ class FeaturesDriftTablePlot:
 
     # Status configurations:
     _STATUS_COLORS = {
-        mlrun.common.schemas.model_monitoring.DriftStatus.NO_DRIFT: "rgb(0,176,80)",  # Green
-        mlrun.common.schemas.model_monitoring.DriftStatus.POSSIBLE_DRIFT: "rgb(255,192,0)",  # Orange
-        mlrun.common.schemas.model_monitoring.DriftStatus.DRIFT_DETECTED: "rgb(208,0,106)",  # Magenta
+        mlrun.common.schemas.model_monitoring.constants.ResultStatusApp.no_detection: "rgb(0,176,80)",  # Green
+        mlrun.common.schemas.model_monitoring.constants.ResultStatusApp.potential_detection: "rgb(255,192,0)",  # Orange
+        mlrun.common.schemas.model_monitoring.constants.ResultStatusApp.detected: "rgb(208,0,106)",  # Magenta
     }
 
     # Font configurations:
@@ -97,7 +122,7 @@ class FeaturesDriftTablePlot:
         inputs_statistics: dict,
         metrics: dict[str, Union[dict, float]],
         drift_results: dict[str, DriftResultType],
-    ) -> str:
+    ) -> _PlotlyTableArtifact:
         """
         Produce the html code of the table plot with the given information and the stored configurations in the class.
 
@@ -106,9 +131,8 @@ class FeaturesDriftTablePlot:
         :param metrics:               The drift detection metrics calculated on the sample set and inputs.
         :param drift_results:         The drift results per feature according to the rules of the monitor.
 
-        :return: The full path to the html file of the plot.
+        :return: The drift table as a plotly artifact.
         """
-        # Plot the drift table:
         figure = self._plot(
             features=list(inputs_statistics.keys()),
             sample_set_statistics=sample_set_statistics,
@@ -116,19 +140,7 @@ class FeaturesDriftTablePlot:
             metrics=metrics,
             drift_results=drift_results,
         )
-
-        # Get its HTML representation:
-        figure_html = figure.to_html()
-
-        # Turn off the table columns dragging by injecting the following JavaScript code:
-        start, end = figure_html.rsplit(";", 1)
-        middle = (
-            ';for (const element of document.getElementsByClassName("table")) '
-            '{element.style.pointerEvents = "none";}'
-        )
-        figure_html = start + middle + end
-
-        return figure_html
+        return _PlotlyTableArtifact(figure=figure, key="drift_table_plot")
 
     def _read_columns_names(self, statistics_dictionary: dict, drift_metrics: dict):
         """
@@ -366,10 +378,10 @@ class FeaturesDriftTablePlot:
             bins = np.array(bins)
             if bins[0] == -sys.float_info.max:
                 bins[0] = bins[1] - (bins[2] - bins[1])
-                hovertext[0] = f"(-∞, {bins[1]})"
+                hovertext[0] = f"(-inf, {bins[1]})"
             if bins[-1] == sys.float_info.max:
                 bins[-1] = bins[-2] + (bins[-2] - bins[-3])
-                hovertext[-1] = f"({bins[-2]}, ∞)"
+                hovertext[-1] = f"({bins[-2]}, inf)"
             # Center the bins (leave the first one):
             bins = 0.5 * (bins[:-1] + bins[1:])
             # Plot the histogram as a line with filled background below it:

mlrun/model_monitoring/helpers.py

@@ -15,6 +15,9 @@
 import datetime
 import typing
 
+import numpy as np
+import pandas as pd
+
 import mlrun
 import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas
@@ -36,10 +39,6 @@ class _BatchDict(typing.TypedDict):
     days: int
 
 
-class _MLRunNoRunsFoundError(Exception):
-    pass
-
-
 def get_stream_path(
     project: str = None,
     function_name: str = mm_constants.MonitoringFunctionNames.STREAM,
@@ -55,8 +54,6 @@ def get_stream_path(
 
     stream_uri = mlrun.get_secret_or_env(
         mlrun.common.schemas.model_monitoring.ProjectSecretKeys.STREAM_PATH
-        if function_name is mm_constants.MonitoringFunctionNames.STREAM
-        else ""
     ) or mlrun.mlconf.get_model_monitoring_file_target_path(
         project=project,
         kind=mlrun.common.schemas.model_monitoring.FileTargetKind.STREAM,
@@ -212,3 +209,45 @@ def update_model_endpoint_last_request(
             endpoint_id=model_endpoint.metadata.uid,
             attributes={EventFieldType.LAST_REQUEST: bumped_last_request},
         )
+
+
+def calculate_inputs_statistics(
+    sample_set_statistics: dict, inputs: pd.DataFrame
+) -> dict:
+    """
+    Calculate the inputs data statistics for drift monitoring purpose.
+
+    :param sample_set_statistics: The sample set (stored end point's dataset to reference) statistics. The bins of the
+                                  histograms of each feature will be used to recalculate the histograms of the inputs.
+    :param inputs:                The inputs to calculate their statistics and later on - the drift with respect to the
+                                  sample set.
+
+    :returns: The calculated statistics of the inputs data.
+    """
+
+    # Use `DFDataInfer` to calculate the statistics over the inputs:
+    inputs_statistics = mlrun.data_types.infer.DFDataInfer.get_stats(
+        df=inputs,
+        options=mlrun.data_types.infer.InferOptions.Histogram,
+    )
+
+    # Recalculate the histograms over the bins that are set in the sample-set of the end point:
+    for feature in inputs_statistics.keys():
+        if feature in sample_set_statistics:
+            counts, bins = np.histogram(
+                inputs[feature].to_numpy(),
+                bins=sample_set_statistics[feature]["hist"][1],
+            )
+            inputs_statistics[feature]["hist"] = [
+                counts.tolist(),
+                bins.tolist(),
+            ]
+        elif "hist" in inputs_statistics[feature]:
+            # Comply with the other common features' histogram length
+            mlrun.common.model_monitoring.helpers.pad_hist(
+                mlrun.common.model_monitoring.helpers.Histogram(
+                    inputs_statistics[feature]["hist"]
+                )
+            )
+
+    return inputs_statistics

mlrun/model_monitoring/stream_processing.py

@@ -24,7 +24,9 @@ import mlrun
 import mlrun.common.model_monitoring.helpers
 import mlrun.config
 import mlrun.datastore.targets
+import mlrun.feature_store as fstore
 import mlrun.feature_store.steps
+import mlrun.model_monitoring.db
 import mlrun.model_monitoring.prometheus
 import mlrun.serving.states
 import mlrun.utils
@@ -36,6 +38,7 @@ from mlrun.common.schemas.model_monitoring.constants import (
     FileTargetKind,
     ModelEndpointTarget,
     ProjectSecretKeys,
+    PrometheusEndpoints,
 )
 from mlrun.utils import logger
 
@@ -183,11 +186,11 @@ class EventStreamProcessor:
         # Step 2 - Filter out events with '-' in the path basename from going forward
         # through the next steps of the stream graph
         def apply_storey_filter_stream_events():
-            # Remove none values from each event
+            # Filter events with Prometheus endpoints path
             graph.add_step(
                 "storey.Filter",
                 "filter_stream_event",
-                _fn="('-' not in event.path.split('/')[-1])",
+                _fn=f"(event.path not in {PrometheusEndpoints.list()})",
                 full_event=True,
             )
 
@@ -587,6 +590,8 @@ class ProcessBeforeParquet(mlrun.feature_store.steps.MapClass):
         for key in [
             EventFieldType.FEATURES,
             EventFieldType.NAMED_FEATURES,
+            EventFieldType.PREDICTION,
+            EventFieldType.NAMED_PREDICTIONS,
         ]:
             event.pop(key, None)
 
@@ -931,6 +936,8 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
     def do(self, event: dict):
         endpoint_id = event[EventFieldType.ENDPOINT_ID]
 
+        feature_values = event[EventFieldType.FEATURES]
+        label_values = event[EventFieldType.PREDICTION]
         # Get feature names and label columns
         if endpoint_id not in self.feature_names:
             endpoint_record = get_endpoint_record(
@@ -966,6 +973,12 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
                     },
                 )
 
+                update_monitoring_feature_set(
+                    endpoint_record=endpoint_record,
+                    feature_names=feature_names,
+                    feature_values=feature_values,
+                )
+
             # Similar process with label columns
             if not label_columns and self._infer_columns_from_data:
                 label_columns = self._infer_label_columns_from_data(event)
@@ -984,6 +997,11 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
                     endpoint_id=endpoint_id,
                     attributes={EventFieldType.LABEL_NAMES: json.dumps(label_columns)},
                 )
+                update_monitoring_feature_set(
+                    endpoint_record=endpoint_record,
+                    feature_names=label_columns,
+                    feature_values=label_values,
+                )
 
             self.label_columns[endpoint_id] = label_columns
             self.feature_names[endpoint_id] = feature_names
@@ -1001,7 +1019,6 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
 
         # Add feature_name:value pairs along with a mapping dictionary of all of these pairs
         feature_names = self.feature_names[endpoint_id]
-        feature_values = event[EventFieldType.FEATURES]
         self._map_dictionary_values(
             event=event,
             named_iters=feature_names,
@@ -1011,7 +1028,6 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
 
         # Add label_name:value pairs along with a mapping dictionary of all of these pairs
         label_names = self.label_columns[endpoint_id]
-        label_values = event[EventFieldType.PREDICTION]
         self._map_dictionary_values(
             event=event,
             named_iters=label_names,
@@ -1137,10 +1153,10 @@ class EventRouting(mlrun.feature_store.steps.MapClass):
         self.project: str = project
 
     def do(self, event):
-        if event.path == "/model-monitoring-metrics":
+        if event.path == PrometheusEndpoints.MODEL_MONITORING_METRICS:
             # Return a parsed Prometheus registry file
             event.body = mlrun.model_monitoring.prometheus.get_registry()
-        elif event.path == "/monitoring-batch-metrics":
+        elif event.path == PrometheusEndpoints.MONITORING_BATCH_METRICS:
             # Update statistical metrics
             for event_metric in event.body:
                 mlrun.model_monitoring.prometheus.write_drift_metrics(
@@ -1149,7 +1165,7 @@ class EventRouting(mlrun.feature_store.steps.MapClass):
                     metric=event_metric[EventFieldType.METRIC],
                     value=event_metric[EventFieldType.VALUE],
                 )
-        elif event.path == "/monitoring-drift-status":
+        elif event.path == PrometheusEndpoints.MONITORING_DRIFT_STATUS:
             # Update drift status
             mlrun.model_monitoring.prometheus.write_drift_status(
                 project=self.project,
@@ -1209,7 +1225,7 @@ def update_endpoint_record(
     endpoint_id: str,
     attributes: dict,
 ):
-    model_endpoint_store = mlrun.model_monitoring.get_model_endpoint_store(
+    model_endpoint_store = mlrun.model_monitoring.get_store_object(
         project=project,
     )
 
@@ -1219,7 +1235,25 @@ def update_endpoint_record(
 
 
 def get_endpoint_record(project: str, endpoint_id: str):
-    model_endpoint_store = mlrun.model_monitoring.get_model_endpoint_store(
+    model_endpoint_store = mlrun.model_monitoring.get_store_object(
         project=project,
     )
     return model_endpoint_store.get_model_endpoint(endpoint_id=endpoint_id)
+
+
+def update_monitoring_feature_set(
+    endpoint_record: dict[str, typing.Any],
+    feature_names: list[str],
+    feature_values: list[typing.Any],
+):
+    monitoring_feature_set = fstore.get_feature_set(
+        endpoint_record[
+            mlrun.common.schemas.model_monitoring.EventFieldType.FEATURE_SET_URI
+        ]
+    )
+    for name, val in zip(feature_names, feature_values):
+        monitoring_feature_set.add_feature(
+            fstore.Feature(name=name, value_type=type(val))
+        )
+
+    monitoring_feature_set.save()

mlrun/model_monitoring/tracking_policy.py

@@ -11,8 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
+import warnings
 from typing import Union
 
 import mlrun.common.schemas.schedule
@@ -55,6 +55,12 @@ class TrackingPolicy(mlrun.model.ModelObj):
                                             writer function, which is a real time nuclio functino, will be deployed
                                             with the same image. By default, the image is mlrun/mlrun.
         """
+        warnings.warn(
+            "The `TrackingPolicy` class is deprecated from version 1.7.0 and is not "
+            "used anymore. It will be removed in 1.9.0.",
+            FutureWarning,
+        )
+
         if isinstance(default_batch_intervals, str):
             default_batch_intervals = (
                 mlrun.common.schemas.schedule.ScheduleCronTrigger.from_crontab(

mlrun/model_monitoring/writer.py

@@ -14,7 +14,6 @@
 
 import datetime
 import json
-from http import HTTPStatus
 from typing import Any, NewType
 
 import pandas as pd
@@ -25,6 +24,7 @@ from v3io_frames.frames_pb2 import IGNORE
 
 import mlrun.common.model_monitoring
 import mlrun.model_monitoring
+import mlrun.model_monitoring.db.stores
 import mlrun.utils.v3io_clients
 from mlrun.common.schemas.model_monitoring.constants import ResultStatusApp, WriterEvent
 from mlrun.common.schemas.notification import NotificationKind, NotificationSeverity
@@ -106,13 +106,11 @@ class ModelMonitoringWriter(StepToDict):
         self.project = project
         self.name = project  # required for the deployment process
         self._v3io_container = self.get_v3io_container(self.name)
-        self._kv_client = self._get_v3io_client().kv
         self._tsdb_client = self._get_v3io_frames_client(self._v3io_container)
         self._custom_notifier = CustomNotificationPusher(
             notification_types=[NotificationKind.slack]
         )
         self._create_tsdb_table()
-        self._kv_schemas = []
 
     @staticmethod
     def get_v3io_container(project_name: str) -> str:
@@ -141,40 +139,10 @@ class ModelMonitoringWriter(StepToDict):
 
     def _update_kv_db(self, event: _AppResultEvent) -> None:
         event = _AppResultEvent(event.copy())
-        endpoint_id = event.pop(WriterEvent.ENDPOINT_ID)
-        app_name = event.pop(WriterEvent.APPLICATION_NAME)
-        metric_name = event.pop(WriterEvent.RESULT_NAME)
-        attributes = {metric_name: json.dumps(event)}
-        self._kv_client.update(
-            container=self._v3io_container,
-            table_path=endpoint_id,
-            key=app_name,
-            attributes=attributes,
+        application_result_store = mlrun.model_monitoring.get_store_object(
+            project=self.project
         )
-        if endpoint_id not in self._kv_schemas:
-            self._generate_kv_schema(endpoint_id)
-        logger.info("Updated V3IO KV successfully", key=app_name)
-
-    def _generate_kv_schema(self, endpoint_id: str):
-        """Generate V3IO KV schema file which will be used by the model monitoring applications dashboard in Grafana."""
-        fields = [
-            {"name": WriterEvent.RESULT_NAME, "type": "string", "nullable": False}
-        ]
-        res = self._kv_client.create_schema(
-            container=self._v3io_container,
-            table_path=endpoint_id,
-            key=WriterEvent.APPLICATION_NAME,
-            fields=fields,
-        )
-        if res.status_code != HTTPStatus.OK.value:
-            raise mlrun.errors.MLRunBadRequestError(
-                f"Couldn't infer schema for endpoint {endpoint_id} which is required for Grafana dashboards"
-            )
-        else:
-            logger.info(
-                "Generated V3IO KV schema successfully", endpoint_id=endpoint_id
-            )
-            self._kv_schemas.append(endpoint_id)
+        application_result_store.write_application_result(event=event)
 
     def _update_tsdb(self, event: _AppResultEvent) -> None:
         event = _AppResultEvent(event.copy())

mlrun/projects/pipelines.py

@@ -412,6 +412,11 @@ def enrich_function_object(
     if decorator:
         decorator(f)
 
+    if project.spec.default_function_node_selector:
+        f.enrich_runtime_spec(
+            project.spec.default_function_node_selector,
+        )
+
     if try_auto_mount:
         if (
             decorator and AutoMountType.is_auto_modifier(decorator)
@@ -608,6 +613,7 @@ class _KFPRunner(_PipelineRunner):
             namespace=namespace,
             artifact_path=artifact_path,
             cleanup_ttl=workflow_spec.cleanup_ttl,
+            timeout=int(mlrun.mlconf.workflows.timeouts.kfp),
         )
 
         # The user provided workflow code might have made changes to function specs that require cleanup
@@ -865,15 +871,21 @@ class _RemoteRunner(_PipelineRunner):
                 )
                 return
 
+            get_workflow_id_timeout = max(
+                int(mlrun.mlconf.workflows.timeouts.remote),
+                int(getattr(mlrun.mlconf.workflows.timeouts, inner_engine.engine)),
+            )
+
             logger.debug(
                 "Workflow submitted, waiting for pipeline run to start",
                 workflow_name=workflow_response.name,
+                get_workflow_id_timeout=get_workflow_id_timeout,
             )
 
             # Getting workflow id from run:
             response = retry_until_successful(
                 1,
-                getattr(mlrun.mlconf.workflows.timeouts, inner_engine.engine),
+                get_workflow_id_timeout,
                 logger,
                 False,
                 run_db.get_workflow_id,