mlrun 1.9.0rc1__py3-none-any.whl → 1.9.0rc3__py3-none-any.whl

mlrun/config.py

@@ -486,6 +486,10 @@ default_config = {
             "iguazio_client_job_cache_ttl": "20 minutes",
             "nuclio_project_deletion_verification_timeout": "300 seconds",
             "nuclio_project_deletion_verification_interval": "5 seconds",
+            "summaries": {
+                # Number of days back to include when calculating the project pipeline summary.
+                "list_pipelines_time_period_in_days": 7,
+            },
         },
         # The API needs to know what is its k8s svc url so it could enrich it in the jobs it creates
         "api_url": "",

mlrun/datastore/sources.py

@@ -971,13 +971,8 @@ class OnlineSource(BaseSourceDriver):
     def set_explicit_ack_mode(function: Function, **extra_arguments) -> dict[str, Any]:
         extra_arguments = extra_arguments or {}
         engine = "sync"
-        if (
-            function.spec
-            and hasattr(function.spec, "graph")
-            and function.spec.graph
-            and function.spec.graph.engine
-        ):
-            engine = function.spec.graph.engine
+        if function.spec and hasattr(function.spec, "graph"):
+            engine = getattr(function.spec.graph, "engine", None) or engine
         if mlrun.mlconf.is_explicit_ack_enabled() and engine == "async":
             extra_arguments["explicit_ack_mode"] = extra_arguments.get(
                 "explicit_ack_mode", "explicitOnly"

mlrun/datastore/storeytargets.py

@@ -109,17 +109,20 @@ class StreamStoreyTarget(storey.StreamTarget):
             raise mlrun.errors.MLRunInvalidArgumentError("StreamTarget requires a path")
 
         _, storage_options = get_url_and_storage_options(uri)
-        endpoint, path = parse_path(uri)
+        _, path = parse_path(uri)
 
         access_key = storage_options.get("v3io_access_key")
-        storage = V3ioDriver(
-            webapi=endpoint or mlrun.mlconf.v3io_api, access_key=access_key
-        )
+
+        if alt_key_name := kwargs.pop("alternative_v3io_access_key", None):
+            if alt_key := mlrun.get_secret_or_env(alt_key_name):
+                access_key = alt_key
+
+        storage = V3ioDriver(access_key=access_key)
 
         if storage_options:
             kwargs["storage"] = storage
         if args:
-            args[0] = endpoint
+            args[0] = path
         if "stream_path" in kwargs:
             kwargs["stream_path"] = path
 
@@ -128,6 +131,7 @@ class StreamStoreyTarget(storey.StreamTarget):
 
 class KafkaStoreyTarget(storey.KafkaTarget):
     def __init__(self, *args, **kwargs):
+        kwargs.pop("alternative_v3io_access_key", None)
         path = kwargs.pop("path")
         attributes = kwargs.pop("attributes", {})
         if path and path.startswith("ds://"):

mlrun/model_monitoring/applications/base.py

@@ -38,21 +38,30 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
 
     For example, :code:`MyApp` below is a simplistic custom application::
 
+        from mlrun.common.schemas.model_monitoring.constants import (
+            ResultKindApp,
+            ResultStatusApp,
+        )
+        from mlrun.model_monitoring.applications import (
+            ModelMonitoringApplicationBase,
+            ModelMonitoringApplicationResult,
+            MonitoringApplicationContext,
+        )
+
+
         class MyApp(ModelMonitoringApplicationBase):
             def do_tracking(
-                self,
-                monitoring_context: mm_context.MonitoringApplicationContext,
+                self, monitoring_context: MonitoringApplicationContext
             ) -> ModelMonitoringApplicationResult:
-                monitoring_context.log_artifact(
-                    TableArtifact(
-                        "sample_df_stats", df=self.dict_to_histogram(sample_df_stats)
-                    )
+                monitoring_context.logger.info(
+                    "Running application",
+                    application_name=monitoring_context.application_name,
                 )
                 return ModelMonitoringApplicationResult(
                     name="data_drift_test",
                     value=0.5,
-                    kind=mm_constant.ResultKindApp.data_drift,
-                    status=mm_constant.ResultStatusApp.detected,
+                    kind=ResultKindApp.data_drift,
+                    status=ResultStatusApp.detected,
                 )
     """
 

mlrun/model_monitoring/applications/context.py

@@ -23,12 +23,13 @@ import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.errors
 import mlrun.feature_store as fstore
+import mlrun.feature_store.feature_set as fs
 import mlrun.features
 import mlrun.serving
 import mlrun.utils
 from mlrun.artifacts import Artifact, DatasetArtifact, ModelArtifact, get_model
 from mlrun.common.model_monitoring.helpers import FeatureStats
-from mlrun.common.schemas import FeatureSet, ModelEndpoint
+from mlrun.common.schemas import ModelEndpoint
 from mlrun.model_monitoring.helpers import (
     calculate_inputs_statistics,
 )
@@ -41,7 +42,6 @@ class _ArtifactsLogger(Protocol):
 
     def log_artifact(self, *args, **kwargs) -> Artifact: ...
     def log_dataset(self, *args, **kwargs) -> DatasetArtifact: ...
-    def log_model(self, *args, **kwargs) -> ModelArtifact: ...
 
 
 class MonitoringApplicationContext:
@@ -59,7 +59,7 @@ class MonitoringApplicationContext:
         model_endpoint_dict: Optional[dict[str, ModelEndpoint]] = None,
         sample_df: Optional[pd.DataFrame] = None,
         feature_stats: Optional[FeatureStats] = None,
-        feature_sets_dict: Optional[dict[str, FeatureSet]] = None,
+        feature_sets_dict: Optional[dict[str, fs.FeatureSet]] = None,
     ) -> None:
         """
         The :code:`MonitoringApplicationContext` object holds all the relevant information for the
@@ -124,9 +124,13 @@ class MonitoringApplicationContext:
         self._model_endpoint: Optional[ModelEndpoint] = (
             model_endpoint_dict.get(self.endpoint_id) if model_endpoint_dict else None
         )
-        self._feature_set: Optional[FeatureSet] = (
+        self._feature_set: Optional[fs.FeatureSet] = (
             feature_sets_dict.get(self.endpoint_id) if feature_sets_dict else None
         )
+        store, _, _ = mlrun.store_manager.get_or_create_store(
+            mlrun.mlconf.artifact_path
+        )
+        self.storage_options = store.get_storage_options()
 
     @classmethod
     def _from_ml_ctx(
@@ -169,7 +173,7 @@ class MonitoringApplicationContext:
         model_endpoint_dict: Optional[dict[str, ModelEndpoint]] = None,
         sample_df: Optional[pd.DataFrame] = None,
         feature_stats: Optional[FeatureStats] = None,
-        feature_sets_dict: Optional[dict[str, FeatureSet]] = None,
+        feature_sets_dict: Optional[dict[str, fs.FeatureSet]] = None,
     ) -> "MonitoringApplicationContext":
         nuclio_logger = graph_context.logger
         artifacts_logger = graph_context.project_obj
@@ -192,13 +196,14 @@ class MonitoringApplicationContext:
         )
 
     def _get_default_labels(self) -> dict[str, str]:
-        return {
+        labels = {
             mlrun_constants.MLRunInternalLabels.runner_pod: socket.gethostname(),
             mlrun_constants.MLRunInternalLabels.producer_type: "model-monitoring-app",
             mlrun_constants.MLRunInternalLabels.app_name: self.application_name,
             mlrun_constants.MLRunInternalLabels.endpoint_id: self.endpoint_id,
             mlrun_constants.MLRunInternalLabels.endpoint_name: self.endpoint_name,
         }
+        return {key: value for key, value in labels.items() if value is not None}
 
     def _add_default_labels(self, labels: Optional[dict[str, str]]) -> dict[str, str]:
         """Add the default labels to logged artifacts labels"""
@@ -221,22 +226,13 @@ class MonitoringApplicationContext:
                     "You can either provide the sample dataframe directly, the model endpoint's details and times, "
                     "or adapt the application's logic to not access the sample dataframe."
                 )
-            feature_set = self.feature_set
-            features = [f"{feature_set.metadata.name}.*"]
-            vector = fstore.FeatureVector(
-                name=f"{self.endpoint_id}_vector",
-                features=features,
-                with_indexes=True,
-            )
-            vector.metadata.tag = self.application_name
-            vector.feature_set_objects = {feature_set.metadata.name: feature_set}
-
-            offline_response = vector.get_offline_features(
+            df = self.feature_set.to_dataframe(
                 start_time=self.start_infer_time,
                 end_time=self.end_infer_time,
-                timestamp_for_filtering=mm_constants.FeatureSetFeatures.time_stamp(),
+                time_column=mm_constants.EventFieldType.TIMESTAMP,
+                storage_options=self.storage_options,
             )
-            self._sample_df = offline_response.to_dataframe().reset_index(drop=True)
+            self._sample_df = df.reset_index(drop=True)
         return self._sample_df
 
     @property
@@ -259,7 +255,7 @@ class MonitoringApplicationContext:
         return self._model_endpoint
 
     @property
-    def feature_set(self) -> FeatureSet:
+    def feature_set(self) -> fs.FeatureSet:
         if not self._feature_set and self.model_endpoint:
             self._feature_set = fstore.get_feature_set(
                 self.model_endpoint.spec.monitoring_feature_set_uri
@@ -329,13 +325,31 @@ class MonitoringApplicationContext:
         upload: Optional[bool] = None,
         labels: Optional[dict[str, str]] = None,
         target_path: Optional[str] = None,
+        unique_per_endpoint: bool = True,
         **kwargs,
     ) -> Artifact:
         """
         Log an artifact.
-        See :func:`~mlrun.projects.MlrunProject.log_artifact` for the documentation.
+
+        .. caution::
+
+            Logging artifacts in every model monitoring window may cause scale issues.
+            This method should be called on special occasions only.
+
+        See :func:`~mlrun.projects.MlrunProject.log_artifact` for the full documentation, except for one
+        new argument:
+
+        :param unique_per_endpoint: by default ``True``, we will log different artifact for each model endpoint,
+                                    set to ``False`` without changing item key will cause artifact override.
         """
         labels = self._add_default_labels(labels)
+        # By default, we want to log different artifact for each model endpoint
+        endpoint_id = labels.get(mlrun_constants.MLRunInternalLabels.endpoint_id, "")
+        if unique_per_endpoint and isinstance(item, str):
+            item = f"{item}-{endpoint_id}" if endpoint_id else item
+        elif unique_per_endpoint:  # isinstance(item, Artifact) is True
+            item.key = f"{item.key}-{endpoint_id}" if endpoint_id else item.key
+
         return self._artifacts_logger.log_artifact(
             item,
             body=body,
@@ -364,13 +378,29 @@ class MonitoringApplicationContext:
         target_path="",
         extra_data=None,
         label_column: Optional[str] = None,
+        unique_per_endpoint: bool = True,
         **kwargs,
     ) -> DatasetArtifact:
         """
         Log a dataset artifact.
-        See :func:`~mlrun.projects.MlrunProject.log_dataset` for the documentation.
+
+        .. caution::
+
+            Logging datasets in every model monitoring window may cause scale issues.
+            This method should be called on special occasions only.
+
+        See :func:`~mlrun.projects.MlrunProject.log_dataset` for the full documentation, except for one
+        new argument:
+
+        :param unique_per_endpoint: by default ``True``, we will log different artifact for each model endpoint,
+                                    set to ``False`` without changing item key will cause artifact override.
         """
         labels = self._add_default_labels(labels)
+        # By default, we want to log different artifact for each model endpoint
+        endpoint_id = labels.get(mlrun_constants.MLRunInternalLabels.endpoint_id, "")
+        if unique_per_endpoint and isinstance(key, str):
+            key = f"{key}-{endpoint_id}" if endpoint_id else key
+
         return self._artifacts_logger.log_dataset(
             key,
             df,
@@ -387,54 +417,3 @@ class MonitoringApplicationContext:
             label_column=label_column,
             **kwargs,
         )
-
-    def log_model(
-        self,
-        key,
-        body=None,
-        framework="",
-        tag="",
-        model_dir=None,
-        model_file=None,
-        algorithm=None,
-        metrics=None,
-        parameters=None,
-        artifact_path=None,
-        upload=None,
-        labels=None,
-        inputs: Optional[list[mlrun.features.Feature]] = None,
-        outputs: Optional[list[mlrun.features.Feature]] = None,
-        feature_vector: Optional[str] = None,
-        feature_weights: Optional[list] = None,
-        training_set=None,
-        label_column=None,
-        extra_data=None,
-        **kwargs,
-    ) -> ModelArtifact:
-        """
-        Log a model artifact.
-        See :func:`~mlrun.projects.MlrunProject.log_model` for the documentation.
-        """
-        labels = self._add_default_labels(labels)
-        return self._artifacts_logger.log_model(
-            key,
-            body=body,
-            framework=framework,
-            tag=tag,
-            model_dir=model_dir,
-            model_file=model_file,
-            algorithm=algorithm,
-            metrics=metrics,
-            parameters=parameters,
-            artifact_path=artifact_path,
-            upload=upload,
-            labels=labels,
-            inputs=inputs,
-            outputs=outputs,
-            feature_vector=feature_vector,
-            feature_weights=feature_weights,
-            training_set=training_set,
-            label_column=label_column,
-            extra_data=extra_data,
-            **kwargs,
-        )

mlrun/model_monitoring/applications/evidently/base.py

@@ -130,17 +130,28 @@ class EvidentlyModelMonitoringApplicationBase(
         monitoring_context: mm_context.MonitoringApplicationContext,
         evidently_object: "Display",
         artifact_name: str,
+        unique_per_endpoint: bool = True,
     ) -> None:
         """
-         Logs an Evidently report or suite as an artifact.
+        Logs an Evidently report or suite as an artifact.
+
+        .. caution::
+
+            Logging Evidently objects in every model monitoring window may cause scale issues.
+            This method should be called on special occasions only.
 
         :param monitoring_context:  (MonitoringApplicationContext) The monitoring context to process.
         :param evidently_object:    (Display) The Evidently display to log, e.g. a report or a test suite object.
         :param artifact_name:       (str) The name for the logged artifact.
+        :param unique_per_endpoint: by default ``True``, we will log different artifact for each model endpoint,
+                                    set to ``False`` without changing item key will cause artifact override.
         """
         evidently_object_html = evidently_object.get_html()
         monitoring_context.log_artifact(
-            artifact_name, body=evidently_object_html.encode("utf-8"), format="html"
+            artifact_name,
+            body=evidently_object_html.encode("utf-8"),
+            format="html",
+            unique_per_endpoint=unique_per_endpoint,
         )
 
     def log_project_dashboard(
@@ -149,14 +160,22 @@ class EvidentlyModelMonitoringApplicationBase(
         timestamp_start: pd.Timestamp,
         timestamp_end: pd.Timestamp,
         artifact_name: str = "dashboard",
+        unique_per_endpoint: bool = True,
     ) -> None:
         """
         Logs an Evidently project dashboard.
 
+        .. caution::
+
+            Logging Evidently dashboards in every model monitoring window may cause scale issues.
+            This method should be called on special occasions only.
+
         :param monitoring_context:  (MonitoringApplicationContext) The monitoring context to process.
         :param timestamp_start:     (pd.Timestamp) The start timestamp for the dashboard data.
         :param timestamp_end:       (pd.Timestamp) The end timestamp for the dashboard data.
         :param artifact_name:       (str) The name for the logged artifact.
+        :param unique_per_endpoint: by default ``True``, we will log different artifact for each model endpoint,
+                                    set to ``False`` without changing item key will cause artifact override.
         """
 
         dashboard_info = self.evidently_project.build_dashboard_info(
@@ -170,5 +189,8 @@ class EvidentlyModelMonitoringApplicationBase(
 
         dashboard_html = file_html_template(params=template_params)
         monitoring_context.log_artifact(
-            artifact_name, body=dashboard_html.encode("utf-8"), format="html"
+            artifact_name,
+            body=dashboard_html.encode("utf-8"),
+            format="html",
+            unique_per_endpoint=unique_per_endpoint,
         )

mlrun/model_monitoring/applications/histogram_data_drift.py

@@ -102,10 +102,10 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
     Each metric is calculated over all the features individually and the mean is taken as the metric value.
     The average of Hellinger and total variance distance is taken as the result.
 
-    The application can log two artifacts:
+    The application can log two artifacts (disabled by default due to performance issues):
 
-    * JSON with the general drift value per feature, produced by default.
-    * Plotly table with the various metrics and histograms per feature (disabled by default due to performance issues).
+    * JSON with the general drift value per feature.
+    * Plotly table with the various metrics and histograms per feature.
 
     This application is deployed by default when calling
     :py:func:`~mlrun.projects.MlrunProject.enable_model_monitoring`.
@@ -134,12 +134,14 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
     def __init__(
         self,
         value_classifier: Optional[ValueClassifier] = None,
-        produce_json_artifact: bool = True,
+        produce_json_artifact: bool = False,
         produce_plotly_artifact: bool = False,
     ) -> None:
         """
-        :param value_classifier: Classifier object that adheres to the :py:class:`~ValueClassifier` protocol.
-                                 If not provided, the default :py:class:`~DataDriftClassifier` is used.
+        :param value_classifier:        Classifier object that adheres to the :py:class:`~ValueClassifier` protocol.
+                                        If not provided, the default :py:class:`~DataDriftClassifier` is used.
+        :param produce_json_artifact:   Whether to produce the JSON artifact or not, ``False`` by default.
+        :param produce_plotly_artifact: Whether to produce the Plotly artifact or not, ``False`` by default.
         """
         self._value_classifier = value_classifier or DataDriftClassifier()
         assert self._REQUIRED_METRICS <= set(

mlrun/model_monitoring/controller.py

@@ -311,10 +311,12 @@ class MonitoringApplicationController:
         mlrun.platforms.iguazio.KafkaOutputStream,
     ]:
         if self._model_monitoring_stream is None:
-            self._model_monitoring_stream = mlrun.model_monitoring.helpers.get_output_stream(
-                project=self.project,
-                function_name=mm_constants.MonitoringFunctionNames.APPLICATION_CONTROLLER,
-                v3io_access_key=self.v3io_access_key,
+            self._model_monitoring_stream = (
+                mlrun.model_monitoring.helpers.get_output_stream(
+                    project=self.project,
+                    function_name=mm_constants.MonitoringFunctionNames.STREAM,
+                    v3io_access_key=self.model_monitoring_access_key,
+                )
             )
         return self._model_monitoring_stream
 

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -11,10 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import asyncio
 from datetime import datetime, timedelta
 from threading import Lock
-from typing import Callable, Literal, Optional, Union
+from typing import Callable, Final, Literal, Optional, Union
 
 import pandas as pd
 import taosws
@@ -24,6 +25,7 @@ from taoswswrap.tdengine_connection import (
 )
 
 import mlrun.common.schemas.model_monitoring as mm_schemas
+import mlrun.common.types
 import mlrun.model_monitoring.db.tsdb.tdengine.schemas as tdengine_schemas
 import mlrun.model_monitoring.db.tsdb.tdengine.stream_graph_steps
 from mlrun.datastore.datastore_profile import DatastoreProfile
@@ -35,6 +37,19 @@ _connection = None
 _connection_lock = Lock()
 
 
+class TDEngineTimestampPrecision(mlrun.common.types.StrEnum):
+    """
+    The timestamp precision for the TDEngine database.
+    For more information, see:
+    https://docs.tdengine.com/tdengine-reference/sql-manual/data-types/#timestamp
+    https://docs.tdengine.com/tdengine-reference/sql-manual/manage-databases/#create-database
+    """
+
+    MILLISECOND = "ms"  # TDEngine's default
+    MICROSECOND = "us"  # MLRun's default
+    NANOSECOND = "ns"
+
+
 class TDEngineConnector(TSDBConnector):
     """
     Handles the TSDB operations when the TSDB connector is of type TDEngine.
@@ -47,12 +62,17 @@ class TDEngineConnector(TSDBConnector):
         self,
         project: str,
         profile: DatastoreProfile,
+        timestamp_precision: TDEngineTimestampPrecision = TDEngineTimestampPrecision.MICROSECOND,
         **kwargs,
     ):
         super().__init__(project=project)
 
         self._tdengine_connection_profile = profile
 
+        self._timestamp_precision: Final = (  # cannot be changed after initialization
+            timestamp_precision
+        )
+
         self._init_super_tables()
 
         self._run_directly = (
@@ -105,7 +125,7 @@ class TDEngineConnector(TSDBConnector):
         """Create the database if it does not exist."""
         self.connection.prefix_statements = []
         self.connection.run(
-            statements=f"CREATE DATABASE IF NOT EXISTS {self.database}",
+            statements=f"CREATE DATABASE IF NOT EXISTS {self.database} PRECISION '{self._timestamp_precision}'",
             timeout=self._timeout,
             retries=self._retries,
         )
@@ -180,6 +200,7 @@ class TDEngineConnector(TSDBConnector):
             columns=columns,
             subtable=table_name,
             values=event,
+            timestamp_precision=self._timestamp_precision,
         )
 
         self.connection.run(

mlrun/model_monitoring/stream_processing.py

@@ -15,8 +15,6 @@
 import datetime
 import typing
 
-import storey
-
 import mlrun
 import mlrun.common.model_monitoring.helpers
 import mlrun.feature_store as fstore
@@ -144,7 +142,7 @@ class EventStreamProcessor:
 
         graph = typing.cast(
             mlrun.serving.states.RootFlowStep,
-            fn.set_topology(mlrun.serving.states.StepKinds.flow),
+            fn.set_topology(mlrun.serving.states.StepKinds.flow, engine="async"),
         )
 
         # split the graph between event with error vs valid event
@@ -264,6 +262,9 @@ class EventStreamProcessor:
                 path=stream_uri,
                 sharding_func=ControllerEvent.ENDPOINT_ID,
                 after="ForwardNOP",
+                # Force using the pipeline key instead of the one in the profile in case of v3io profile.
+                # In case of Kafka, this parameter will be ignored.
+                alternative_v3io_access_key="V3IO_ACCESS_KEY",
             )
 
         apply_push_controller_stream(controller_stream_uri)
@@ -342,7 +343,8 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
             logger.debug(
                 "Skipped nop event inside of ProcessEndpointEvent", event=event
             )
-            return storey.Event(body=[event])
+            full_event.body = [event]
+            return full_event
         # Getting model version and function uri from event
         # and use them for retrieving the endpoint_id
         function_uri = full_event.body.get(EventFieldType.FUNCTION_URI)
@@ -382,9 +384,6 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
             # Set time for the first request of the current endpoint
             self.first_request[endpoint_id] = timestamp
 
-        # Set time for the last reqeust of the current endpoint
-        self.last_request[endpoint_id] = timestamp
-
         if not self.is_valid(
             validation_function=is_not_none,
             field=request_id,
@@ -411,7 +410,7 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
             return None
 
         # Convert timestamp to a datetime object
-        timestamp = datetime.datetime.fromisoformat(timestamp)
+        timestamp_obj = datetime.datetime.fromisoformat(timestamp)
 
         # Separate each model invocation into sub events that will be stored as dictionary
         # in list of events. This list will be used as the body for the storey event.
@@ -452,16 +451,16 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
                     EventFieldType.FUNCTION_URI: function_uri,
                     EventFieldType.ENDPOINT_NAME: event.get(EventFieldType.MODEL),
                     EventFieldType.MODEL_CLASS: model_class,
-                    EventFieldType.TIMESTAMP: timestamp,
+                    EventFieldType.TIMESTAMP: timestamp_obj,
                     EventFieldType.ENDPOINT_ID: endpoint_id,
                     EventFieldType.REQUEST_ID: request_id,
                     EventFieldType.LATENCY: latency,
                     EventFieldType.FEATURES: feature,
                     EventFieldType.PREDICTION: prediction,
                     EventFieldType.FIRST_REQUEST: self.first_request[endpoint_id],
-                    EventFieldType.LAST_REQUEST: self.last_request[endpoint_id],
+                    EventFieldType.LAST_REQUEST: timestamp,
                     EventFieldType.LAST_REQUEST_TIMESTAMP: mlrun.utils.enrich_datetime_with_tz_info(
-                        self.last_request[endpoint_id]
+                        timestamp
                     ).timestamp(),
                     EventFieldType.LABELS: event.get(EventFieldType.LABELS, {}),
                     EventFieldType.METRICS: event.get(EventFieldType.METRICS, {}),
@@ -475,8 +474,9 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
 
         # Create a storey event object with list of events, based on endpoint_id which will be used
         # in the upcoming steps
-        storey_event = storey.Event(body=events, key=endpoint_id)
-        return storey_event
+        full_event.key = endpoint_id
+        full_event.body = events
+        return full_event
 
     def resume_state(self, endpoint_id, endpoint_name):
         # Make sure process is resumable, if process fails for any reason, be able to pick things up close to where we
@@ -489,6 +489,7 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
                     project=self.project,
                     endpoint_id=endpoint_id,
                     name=endpoint_name,
+                    tsdb_metrics=False,
                 )
                 .flat_dict()
             )
@@ -500,10 +501,6 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
                 if first_request:
                     self.first_request[endpoint_id] = first_request
 
-                last_request = endpoint_record.get(EventFieldType.LAST_REQUEST)
-                if last_request:
-                    self.last_request[endpoint_id] = last_request
-
             # add endpoint to endpoints set
             self.endpoints.add(endpoint_id)
 
@@ -616,6 +613,7 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
                     project=self.project,
                     endpoint_id=endpoint_id,
                     name=event[EventFieldType.ENDPOINT_NAME],
+                    tsdb_metrics=False,
                 )
                 .flat_dict()
             )
@@ -689,6 +687,7 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
                     project=self.project,
                     endpoint_id=endpoint_id,
                     name=event[EventFieldType.ENDPOINT_NAME],
+                    tsdb_metrics=False,
                 )
                 .flat_dict()
             )

mlrun/runtimes/nuclio/function.py

@@ -577,13 +577,9 @@ class RemoteRuntime(KubeResource):
             access_key = self._resolve_v3io_access_key()
         engine = "sync"
         explicit_ack_mode = kwargs.pop("explicit_ack_mode", None)
-        if (
-            self.spec
-            and hasattr(self.spec, "graph")
-            and self.spec.graph
-            and self.spec.graph.engine
-        ):
-            engine = self.spec.graph.engine
+        if self.spec and hasattr(self.spec, "graph"):
+            engine = getattr(self.spec.graph, "engine", None) or engine
+
         if mlrun.mlconf.is_explicit_ack_enabled() and engine == "async":
             explicit_ack_mode = explicit_ack_mode or "explicitOnly"
 

mlrun/runtimes/nuclio/serving.py

@@ -271,7 +271,8 @@ class ServingRuntime(RemoteRuntime):
                    can specify special router class and router arguments
 
           flow   - workflow (DAG) with a chain of states
-                   flow support "sync" and "async" engines, branches are not allowed in sync mode
+                   flow supports both "sync" and "async" engines, with "async" being the default.
+                   Branches are not allowed in sync mode.
                    when using async mode calling state.respond() will mark the state as the
                    one which generates the (REST) call response
 
@@ -300,7 +301,7 @@ class ServingRuntime(RemoteRuntime):
                 step = RouterStep(class_name=class_name, class_args=class_args)
             self.spec.graph = step
         elif topology == StepKinds.flow:
-            self.spec.graph = RootFlowStep(engine=engine)
+            self.spec.graph = RootFlowStep(engine=engine or "async")
         else:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 f"unsupported topology {topology}, use 'router' or 'flow'"

mlrun/serving/states.py

@@ -415,15 +415,18 @@ class BaseStep(ModelObj):
         steps: list[Union[str, StepToDict, dict[str, Any]]],
         force: bool = False,
     ):
-        """set list of steps as downstream from this step, in the order specified. This will overwrite any existing
+        """
+        Set list of steps as downstream from this step, in the order specified. This will overwrite any existing
         downstream steps.
 
         :param steps: list of steps to follow this one
         :param force: whether to overwrite existing downstream steps. If False, this method will fail if any downstream
-        steps have already been defined. Defaults to False.
+                      steps have already been defined. Defaults to False.
+
         :return: the last step added to the flow
 
-        example:
+        example::
+
             The below code sets the downstream nodes of step1 by using a list of steps (provided to `set_flow()`) and a
             single step (provided to `to()`), resulting in the graph (step1 -> step2 -> step3 -> step4).
             Notice that using `force=True` is required in case step1 already had downstream nodes (e.g. if the existing
@@ -1833,7 +1836,7 @@ def params_to_step(
     class_args = class_args or {}
 
     if isinstance(class_name, QueueStep):
-        if not name or class_name.name:
+        if not (name or class_name.name):
             raise MLRunInvalidArgumentError("queue name must be specified")
 
         step = class_name
@@ -1854,7 +1857,11 @@ def params_to_step(
     elif class_name and hasattr(class_name, "to_dict"):
         struct = class_name.to_dict()
         kind = struct.get("kind", StepKinds.task)
-        name = name or struct.get("name", struct.get("class_name"))
+        name = (
+            name
+            or struct.get("name", struct.get("class_name"))
+            or class_name.to_dict(["name"]).get("name")
+        )
         cls = classes_map.get(kind, RootFlowStep)
         step = cls.from_dict(struct)
         step.function = function

mlrun/utils/helpers.py

@@ -41,6 +41,7 @@ import inflection
 import numpy as np
 import packaging.version
 import pandas
+import pytz
 import semver
 import yaml
 from dateutil import parser
@@ -1131,21 +1132,83 @@ def get_workflow_url(
     return url
 
 
-def get_kfp_project_filter(project_name: str) -> str:
+def get_kfp_list_runs_filter(
+    project_name: Optional[str] = None,
+    end_date: Optional[str] = None,
+    start_date: Optional[str] = None,
+) -> str:
     """
-    Generates a filter string for KFP runs, using a substring predicate
-    on the run's 'name' field. This is used as a heuristic to retrieve runs that are associated
-    with a specific project. The 'op: 9' operator indicates that the filter checks if the
-    project name appears as a substring in the run's name, ensuring that we can identify
-    runs belonging to the desired project.
+    Generates a filter for listing Kubeflow Pipelines (KFP) runs.
+
+    :param project_name: The name of the project. If "*", it won't filter by project.
+    :param end_date: The latest creation date for filtering runs (ISO 8601 format).
+    :param start_date: The earliest creation date for filtering runs (ISO 8601 format).
+    :return: A JSON-formatted filter string for KFP.
     """
-    is_substring_op = 9
-    project_name_filter = {
-        "predicates": [
-            {"key": "name", "op": is_substring_op, "string_value": project_name}
-        ]
-    }
-    return json.dumps(project_name_filter)
+
+    # KFP filter operation codes
+    kfp_less_than_or_equal_op = 7  # '<='
+    kfp_greater_than_or_equal_op = 5  # '>='
+    kfp_substring_op = 9  # Substring match
+
+    filters = {"predicates": []}
+
+    if end_date:
+        filters["predicates"].append(
+            {
+                "key": "created_at",
+                "op": kfp_less_than_or_equal_op,
+                "timestamp_value": end_date,
+            }
+        )
+
+    if project_name and project_name != "*":
+        filters["predicates"].append(
+            {
+                "key": "name",
+                "op": kfp_substring_op,
+                "string_value": project_name,
+            }
+        )
+    if start_date:
+        filters["predicates"].append(
+            {
+                "key": "created_at",
+                "op": kfp_greater_than_or_equal_op,
+                "timestamp_value": start_date,
+            }
+        )
+    return json.dumps(filters)
+
+
+def validate_and_convert_date(date_input: str) -> str:
+    """
+    Converts any recognizable date string into a standardized RFC 3339 format.
+    :param date_input: A date string in a recognizable format.
+    """
+    try:
+        dt_object = parser.parse(date_input)
+        if dt_object.tzinfo is not None:
+            # Convert to UTC if it's in a different timezone
+            dt_object = dt_object.astimezone(pytz.utc)
+        else:
+            # If no timezone info is present, assume it's in local time
+            local_tz = pytz.timezone("UTC")
+            dt_object = local_tz.localize(dt_object)
+
+        # Convert the datetime object to an RFC 3339-compliant string.
+        # RFC 3339 requires timestamps to be in ISO 8601 format with a 'Z' suffix for UTC time.
+        # The isoformat() method adds a "+00:00" suffix for UTC by default,
+        # so we replace it with "Z" to ensure compliance.
+        formatted_date = dt_object.isoformat().replace("+00:00", "Z")
+        formatted_date = formatted_date.rstrip("Z") + "Z"
+
+        return formatted_date
+    except (ValueError, OverflowError) as e:
+        raise ValueError(
+            f"Invalid date format: {date_input}."
+            f" Date format must adhere to the RFC 3339 standard (e.g., 'YYYY-MM-DDTHH:MM:SSZ' for UTC)."
+        ) from e
 
 
 def are_strings_in_exception_chain_messages(
@@ -2156,7 +2219,10 @@ class Workflow:
     def _get_workflow_manifest(
         workflow_id: str,
     ) -> typing.Optional[mlrun_pipelines.models.PipelineManifest]:
-        kfp_client = mlrun_pipelines.utils.get_client(mlrun.mlconf.kfp_url)
+        kfp_client = mlrun_pipelines.utils.get_client(
+            url=mlrun.mlconf.kfp_url,
+            namespace=mlrun.mlconf.namespace,
+        )
 
         # arbitrary timeout of 5 seconds, the workflow should be done by now
         kfp_run = kfp_client.wait_for_run_completion(workflow_id, 5)

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "b08bde55bcb7fde423002ec145ace1e99e677955",
-  "version": "1.9.0-rc1"
+  "git_commit": "847c2d4b159d98e5e456cbc4898272b9dac77d65",
+  "version": "1.9.0-rc3"
 }

{mlrun-1.9.0rc1.dist-info → mlrun-1.9.0rc3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.9.0rc1
+Version: 1.9.0rc3
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -44,16 +44,15 @@ Requires-Dist: semver~=3.0
 Requires-Dist: dependency-injector~=4.41
 Requires-Dist: fsspec<2024.7,>=2023.9.2
 Requires-Dist: v3iofs~=0.1.17
-Requires-Dist: storey~=1.8.9
+Requires-Dist: storey~=1.9.0
 Requires-Dist: inflection~=0.5.0
 Requires-Dist: python-dotenv~=1.0
 Requires-Dist: setuptools>=75.2
 Requires-Dist: deprecated~=1.2
 Requires-Dist: jinja2>=3.1.3,~=3.1
 Requires-Dist: orjson<4,>=3.9.15
-Requires-Dist: mlrun-pipelines-kfp-common~=0.3.12
-Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.3.9; python_version < "3.11"
-Requires-Dist: mlrun-pipelines-kfp-v2~=0.3.8; python_version >= "3.11"
+Requires-Dist: mlrun-pipelines-kfp-common~=0.4.1
+Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.4.0
 Requires-Dist: docstring_parser~=0.16
 Requires-Dist: aiosmtplib~=3.0
 Provides-Extra: s3
@@ -100,11 +99,11 @@ Requires-Dist: ossfs==2023.12.0; extra == "alibaba-oss"
 Requires-Dist: oss2==2.18.1; extra == "alibaba-oss"
 Provides-Extra: tdengine
 Requires-Dist: taos-ws-py==0.3.2; extra == "tdengine"
-Requires-Dist: taoswswrap~=0.3.4; extra == "tdengine"
+Requires-Dist: taoswswrap~=0.3.5; extra == "tdengine"
 Provides-Extra: snowflake
 Requires-Dist: snowflake-connector-python~=3.7; extra == "snowflake"
 Provides-Extra: kfp18
-Requires-Dist: mlrun_pipelines_kfp_v1_8[kfp]>=0.3.2; python_version < "3.11" and extra == "kfp18"
+Requires-Dist: mlrun_pipelines_kfp_v1_8[kfp]>=0.4.0; python_version < "3.11" and extra == "kfp18"
 Provides-Extra: api
 Requires-Dist: uvicorn~=0.32.1; extra == "api"
 Requires-Dist: dask-kubernetes~=0.11.0; extra == "api"
@@ -120,7 +119,7 @@ Requires-Dist: timelength~=1.1; extra == "api"
 Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "api"
 Requires-Dist: aiosmtplib~=3.0; extra == "api"
 Requires-Dist: pydantic<2,>=1; extra == "api"
-Requires-Dist: mlrun-pipelines-kfp-v1-8[kfp]~=0.3.9; python_version < "3.11" and extra == "api"
+Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.4.0; extra == "api"
 Requires-Dist: grpcio~=1.70.0; extra == "api"
 Provides-Extra: all
 Requires-Dist: adlfs==2023.9.0; extra == "all"
@@ -153,7 +152,7 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "all"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "all"
 Requires-Dist: sqlalchemy~=1.4; extra == "all"
 Requires-Dist: taos-ws-py==0.3.2; extra == "all"
-Requires-Dist: taoswswrap~=0.3.4; extra == "all"
+Requires-Dist: taoswswrap~=0.3.5; extra == "all"
 Provides-Extra: complete
 Requires-Dist: adlfs==2023.9.0; extra == "complete"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete"
@@ -185,7 +184,7 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "complete"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "complete"
 Requires-Dist: sqlalchemy~=1.4; extra == "complete"
 Requires-Dist: taos-ws-py==0.3.2; extra == "complete"
-Requires-Dist: taoswswrap~=0.3.4; extra == "complete"
+Requires-Dist: taoswswrap~=0.3.5; extra == "complete"
 Provides-Extra: complete-api
 Requires-Dist: adlfs==2023.9.0; extra == "complete-api"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete-api"
@@ -216,7 +215,7 @@ Requires-Dist: igz-mgmt~=0.4.1; extra == "complete-api"
 Requires-Dist: kafka-python~=2.0; extra == "complete-api"
 Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "complete-api"
 Requires-Dist: mlflow~=2.16; extra == "complete-api"
-Requires-Dist: mlrun-pipelines-kfp-v1-8[kfp]~=0.3.9; python_version < "3.11" and extra == "complete-api"
+Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.4.0; extra == "complete-api"
 Requires-Dist: msrest~=0.6.21; extra == "complete-api"
 Requires-Dist: objgraph~=3.6; extra == "complete-api"
 Requires-Dist: oss2==2.18.1; extra == "complete-api"
@@ -230,7 +229,7 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "complete-api"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "complete-api"
 Requires-Dist: sqlalchemy~=1.4; extra == "complete-api"
 Requires-Dist: taos-ws-py==0.3.2; extra == "complete-api"
-Requires-Dist: taoswswrap~=0.3.4; extra == "complete-api"
+Requires-Dist: taoswswrap~=0.3.5; extra == "complete-api"
 Requires-Dist: timelength~=1.1; extra == "complete-api"
 Requires-Dist: uvicorn~=0.32.1; extra == "complete-api"
 Dynamic: author

{mlrun-1.9.0rc1.dist-info → mlrun-1.9.0rc3.dist-info}/RECORD

@@ -1,6 +1,6 @@
 mlrun/__init__.py,sha256=Cqm9U9eCEdLpMejhU2BEhubu0mHL71igJJIwYa738EA,7450
 mlrun/__main__.py,sha256=0NDzPf9VFRO8KFfGgb8mkGUPIDS285aASV8Hbxs-ND0,45920
-mlrun/config.py,sha256=vjlKlex9TV1Gfgix4iOM_nzX5wN-YuV6xXPmh0JDWy4,71932
+mlrun/config.py,sha256=JptQ5cano4qGdSt8sFiEMQpT96Wr9Rv4A9q_cyMJgOo,72127
 mlrun/errors.py,sha256=LkcbXTLANGdsgo2CRX2pdbyNmt--lMsjGv0XZMgP-Nc,8222
 mlrun/execution.py,sha256=FUktsD3puSFjc3LZJU35b-OmFBrBPBNntViCLQVuwnk,50008
 mlrun/features.py,sha256=ReBaNGsBYXqcbgI012n-SO_j6oHIbk_Vpv0CGPXbUmo,15842
@@ -95,11 +95,11 @@ mlrun/datastore/inmem.py,sha256=IsM83nn-3CqmGdLzim7i9ZmJwG6ZGhBZGN6_hszWZnE,2951
 mlrun/datastore/redis.py,sha256=QeNMkSz3zQXiXZhFUZcEtViqqbUysGJditbqe5M-J48,5682
 mlrun/datastore/s3.py,sha256=lD4Fs69rwMeISovZzOxRdz_z9FuffysTdjJA9ybdnLA,9262
 mlrun/datastore/snowflake_utils.py,sha256=Wohvnlmq8j1d98RCaknll-iWdZZpSlCrKhUOEy0_-CA,1483
-mlrun/datastore/sources.py,sha256=KQp1nNN7TcaewFm3It03H1R28uzlWGZDDHJyqiT--vw,49062
+mlrun/datastore/sources.py,sha256=juPTIDpxHxbRBoTMPEG1V-6bgR3E3ufCir-Dnq_SFyg,48975
 mlrun/datastore/spark_udf.py,sha256=NnnB3DZxZb-rqpRy7b-NC7QWXuuqFn3XkBDc86tU4mQ,1498
 mlrun/datastore/spark_utils.py,sha256=_AsVoU5Ix_-W7Gyq8io8V-2GTk0m8THJNDP3WGGaWJY,2865
 mlrun/datastore/store_resources.py,sha256=PFOMrZ6KH6hBOb0PiO-cHx_kv0UpHu5P2t8_mrR-lS4,6842
-mlrun/datastore/storeytargets.py,sha256=g5zAdizdFkcESoVGxbKWC11ZiXFgM77UL4642G32JaU,6459
+mlrun/datastore/storeytargets.py,sha256=dSy9wr4IyxrIE1GHBxzVEeEY1sdU66s4w-oUuaIfa2U,6620
 mlrun/datastore/targets.py,sha256=7qLf26BDH3qYTHOR7TSP0tUMPBhYOkaaOwffUBxgqY0,81201
 mlrun/datastore/utils.py,sha256=CbKbDI6CdFRCqyAXe-jykVvN_GH6R0JkxIQFAogR2GA,10604
 mlrun/datastore/v3io.py,sha256=QSYBORRLcJTeM9mt0EaWzyLcdmzrPkqrF7k5uLTam5U,8209
@@ -219,20 +219,20 @@ mlrun/launcher/local.py,sha256=775HY-8S9LFUX5ubGXrLO0N1lVh8bn-DHFmNYuNqQPA,11451
 mlrun/launcher/remote.py,sha256=rLJW4UAnUT5iUb4BsGBOAV3K4R29a0X4lFtRkVKlyYU,7709
 mlrun/model_monitoring/__init__.py,sha256=ELy7njEtZnz09Dc6PGZSFFEGtnwI15bJNWM3Pj4_YIs,753
 mlrun/model_monitoring/api.py,sha256=LU58dzE4QZiMH23lgiqfI__3m2E3eEZP-DQe2ioUSwM,28317
-mlrun/model_monitoring/controller.py,sha256=fpCfM2wrd9sk-GzmTsqqf1jL1DNgvQhseRRqI-MvMFU,36803
+mlrun/model_monitoring/controller.py,sha256=m4Zx_NQ0C-A7WtjBoXnqBmS11RRtLvBaFgbFbIgrdVc,36847
 mlrun/model_monitoring/features_drift_table.py,sha256=c6GpKtpOJbuT1u5uMWDL_S-6N4YPOmlktWMqPme3KFY,25308
 mlrun/model_monitoring/helpers.py,sha256=8QsoYRPOVSnR3Lcv99m4XYrp_cR6hSqBUflYSOkJmFQ,21019
-mlrun/model_monitoring/stream_processing.py,sha256=4M0H4txMlsC2Q5iKTPp992KWoNPAJjPHj9rqWhXbl8w,33321
+mlrun/model_monitoring/stream_processing.py,sha256=Gu3TQzYoNjbreZYI73-F49QpYrod9RZOyGSgininBsA,33373
 mlrun/model_monitoring/tracking_policy.py,sha256=PBIGrUYWrwcE5gwXupBIVzOb0QRRwPJsgQm_yLGQxB4,5595
 mlrun/model_monitoring/writer.py,sha256=ibbhvfSHb8Reqlb7RGFEAUNM4iTyK1gk8-2m46mP6VM,8428
 mlrun/model_monitoring/applications/__init__.py,sha256=xDBxkBjl-whHSG_4t1mLkxiypLH-fzn8TmAW9Mjo2uI,759
 mlrun/model_monitoring/applications/_application_steps.py,sha256=PxULZznKW66Oq-fKaraOAbsTuGnV0zgXh6_91wX3KUo,8367
-mlrun/model_monitoring/applications/base.py,sha256=7XL12idItWkoE3CJ_48F6cwVx5pJH3bgfG92hb8LcN8,24872
-mlrun/model_monitoring/applications/context.py,sha256=DKUDOfN4iY5wpOMjfsarx4pVN9A1sORyu7y2EEKEvMs,16964
-mlrun/model_monitoring/applications/histogram_data_drift.py,sha256=09t0tfC35W0SeJA3fzN29pJiB6G-V_8GlcvULVq6H9Q,15179
+mlrun/model_monitoring/applications/base.py,sha256=f73LycKUG85invl6l7V4MRiRd1bx8jmepayrpwpr3c0,25131
+mlrun/model_monitoring/applications/context.py,sha256=VfyPCIdO4z73uqFcJs87jzSI4PatX5N5Xicg8Ye1Bag,16968
+mlrun/model_monitoring/applications/histogram_data_drift.py,sha256=2qgfFmrpHf-x0_EaHD-0T28piwSQzw-HH71aV1GwbZs,15389
 mlrun/model_monitoring/applications/results.py,sha256=_qmj6TWT0SR2bi7gUyRKBU418eGgGoLW2_hTJ7S-ock,5782
 mlrun/model_monitoring/applications/evidently/__init__.py,sha256=-DqdPnBSrjZhFvKOu_Ie3MiFvlur9sPTZpZ1u0_1AE8,690
-mlrun/model_monitoring/applications/evidently/base.py,sha256=_n_2CCQL-fC6hGUZSCLZxZuvXqMqjDHSFX0Giok8HZw,6793
+mlrun/model_monitoring/applications/evidently/base.py,sha256=3loXT5gOn5v7j2CiXDQnA0cQXroItBSso6aU0t_pzBs,7851
 mlrun/model_monitoring/db/__init__.py,sha256=r47xPGZpIfMuv8J3PQCZTSqVPMhUta4sSJCZFKcS7FM,644
 mlrun/model_monitoring/db/_schedules.py,sha256=RWn4wtKsIXg668gMLpxO9I8GlkxvPSaA5y7w-wFDcgE,9048
 mlrun/model_monitoring/db/_stats.py,sha256=VVMWLMqG3Us3ozBkLaokJF22Ewv8WKmVE1-OvS_g9vA,6943
@@ -242,7 +242,7 @@ mlrun/model_monitoring/db/tsdb/helpers.py,sha256=0oUXc4aUkYtP2SGP6jTb3uPPKImIUsV
 mlrun/model_monitoring/db/tsdb/tdengine/__init__.py,sha256=vgBdsKaXUURKqIf3M0y4sRatmSVA4CQiJs7J5dcVBkQ,620
 mlrun/model_monitoring/db/tsdb/tdengine/schemas.py,sha256=EslhaR65jfeNdD5Ibk-3Hb4e5r5qYPfHb9rTChX3sG0,12689
 mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py,sha256=Uadj0UvAmln2MxDWod-kAzau1uNlqZh981rPhbUH_5M,2857
-mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=5jgimfu2-omy8Cnnby7GpgB_MWEp9mmLX0zpbGC2JZ8,37934
+mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=rpE-RB5sqe5GAkd9MZB4JFzSP1skxs0loC5hhAkNgKk,38747
 mlrun/model_monitoring/db/tsdb/v3io/__init__.py,sha256=aL3bfmQsUQ-sbvKGdNihFj8gLCK3mSys0qDcXtYOwgc,616
 mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py,sha256=_-zo9relCDtjGgievxAcAP9gVN9nDWs8BzGtFwTjb9M,6284
 mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=IzdThNwWMBWo0D0VzXV-WVvGg-z7Y9e8ke8_LYJTeVA,46214
@@ -292,9 +292,9 @@ mlrun/runtimes/mpijob/abstract.py,sha256=JGMjcJ4dvpJbctF6psU9UvYyNCutMxTMgBQeTlz
 mlrun/runtimes/mpijob/v1.py,sha256=1XQZC7AIMGX_AQCbApcwpH8I7y39-v0v2O35MvxjXoo,3213
 mlrun/runtimes/nuclio/__init__.py,sha256=gx1kizzKv8pGT5TNloN1js1hdbxqDw3rM90sLVYVffY,794
 mlrun/runtimes/nuclio/api_gateway.py,sha256=vH9ClKVP4Mb24rvA67xPuAvAhX-gAv6vVtjVxyplhdc,26969
-mlrun/runtimes/nuclio/function.py,sha256=1EFdGFqlyEfPUVK4Rhh8zWUrff7MNKaHrg7V-bejewg,54618
+mlrun/runtimes/nuclio/function.py,sha256=8wzAFYCpSs0KoGPSN6DC19smSfYh8dVqDUhpicr8sJ0,54540
 mlrun/runtimes/nuclio/nuclio.py,sha256=sLK8KdGO1LbftlL3HqPZlFOFTAAuxJACZCVl1c0Ha6E,2942
-mlrun/runtimes/nuclio/serving.py,sha256=qetAyl-nfn8SWp7KyNgRtMNUVcX_q75SY9dLZP0uH6o,33365
+mlrun/runtimes/nuclio/serving.py,sha256=d0nzPALUYXO4fKFFhxW3hY-_NU-ZhBLWXa2vWIetBRI,33434
 mlrun/runtimes/nuclio/application/__init__.py,sha256=rRs5vasy_G9IyoTpYIjYDafGoL6ifFBKgBtsXn31Atw,614
 mlrun/runtimes/nuclio/application/application.py,sha256=VPX-ruYQJ7-7yd5c2sWdF4U5JCGSS3kYjUfOgev6l_Y,29186
 mlrun/runtimes/nuclio/application/reverse_proxy.go,sha256=lEHH74vr2PridIHp1Jkc_NjkrWb5b6zawRrNxHQhwGU,2913
@@ -306,7 +306,7 @@ mlrun/serving/remote.py,sha256=gxJkj_J3j-sZcVUbUzbAmJafP6t6y4NVFsu0kWmYngA,18818
 mlrun/serving/routers.py,sha256=SY6AsaiSnh8ssXq8hQE2z9MYapOxFOFJBx9QomiZMO8,53915
 mlrun/serving/server.py,sha256=KiNhW0nTV5STZPzR6kEAUFVzCCAX8qv0g9AoCopARrM,23429
 mlrun/serving/serving_wrapper.py,sha256=R670-S6PX_d5ER6jiHtRvacuPyFzQH0mEf2K0sBIIOM,836
-mlrun/serving/states.py,sha256=UWiE85MB_SK3rgzWgNqQU2MKeyN2yF2BCvMcMAqLMTs,73247
+mlrun/serving/states.py,sha256=PaVdgZ8GyE8bxr_-RvuSUINwHuPypJmqaQs27aEPweo,73367
 mlrun/serving/utils.py,sha256=k2EIYDWHUGkE-IBI6T0UNT32fw-KySsccIJM_LObI00,4171
 mlrun/serving/v1_serving.py,sha256=c6J_MtpE-Tqu00-6r4eJOCO6rUasHDal9W2eBIcrl50,11853
 mlrun/serving/v2_serving.py,sha256=b3C5Utv2_AOPrH_hPi3NarjNbAK3kRoeIfqMU4qNuUo,25362
@@ -321,7 +321,7 @@ mlrun/utils/azure_vault.py,sha256=IEFizrDGDbAaoWwDr1WoA88S_EZ0T--vjYtY-i0cvYQ,34
 mlrun/utils/clones.py,sha256=yXOeuLtgIiKZdmjeKK0Z_vIrH19ds5JuoJaCeDjhwOo,7516
 mlrun/utils/condition_evaluator.py,sha256=-nGfRmZzivn01rHTroiGY4rqEv8T1irMyhzxEei-sKc,1897
 mlrun/utils/db.py,sha256=blQgkWMfFH9lcN4sgJQcPQgEETz2Dl_zwbVA0SslpFg,2186
-mlrun/utils/helpers.py,sha256=FflaMvt_8zYDsW5zHG8s1WWda3CbERTCMJUGyziWIjg,74587
+mlrun/utils/helpers.py,sha256=KCvosDXfFNpxnoRAj33BbmMsQqK4wz1Wfj0s9MwOvSY,76782
 mlrun/utils/http.py,sha256=t6FrXQstZm9xVVjxqIGiLzrwZNCR4CSienSOuVgNIcI,8706
 mlrun/utils/logger.py,sha256=RG0m1rx6gfkJ-2C1r_p41MMpPiaDYqaYM2lYHDlNZEU,14767
 mlrun/utils/regex.py,sha256=jbR7IiOp6OO0mg9Fl_cVZCpWb9fL9nTPONCUxCDNWXg,5201
@@ -340,11 +340,11 @@ mlrun/utils/notifications/notification/mail.py,sha256=ZyJ3eqd8simxffQmXzqd3bgbAq
 mlrun/utils/notifications/notification/slack.py,sha256=eQvmctTh6wIG5xVOesLLV9S1-UUCu5UEQ9JIJOor3ts,7183
 mlrun/utils/notifications/notification/webhook.py,sha256=NeyIMSBojjjTJaUHmPbxMByp34GxYkl1-16NqzU27fU,4943
 mlrun/utils/version/__init__.py,sha256=7kkrB7hEZ3cLXoWj1kPoDwo4MaswsI2JVOBpbKgPAgc,614
-mlrun/utils/version/version.json,sha256=WlaePndVtYxwhDSf-ETGiiy3I87M637FDZBAAl1SjvM,88
+mlrun/utils/version/version.json,sha256=MRGJZSpyMyIv0X3-Zt8_WbXo0TFncwEGqRiBH0gbp74,88
 mlrun/utils/version/version.py,sha256=eEW0tqIAkU9Xifxv8Z9_qsYnNhn3YH7NRAfM-pPLt1g,1878
-mlrun-1.9.0rc1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mlrun-1.9.0rc1.dist-info/METADATA,sha256=sRPSncr_i_JhtIKIVROkuR5UMqd6VMEcav64CqOf1_0,26078
-mlrun-1.9.0rc1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-mlrun-1.9.0rc1.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
-mlrun-1.9.0rc1.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.9.0rc1.dist-info/RECORD,,
+mlrun-1.9.0rc3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mlrun-1.9.0rc3.dist-info/METADATA,sha256=KpnUgxmI3MsHDf7CXAwXfNam8sM3TM2QOQyvWI7jxXs,25915
+mlrun-1.9.0rc3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+mlrun-1.9.0rc3.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
+mlrun-1.9.0rc3.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.9.0rc3.dist-info/RECORD,,