mlrun 1.8.0rc46__py3-none-any.whl → 1.8.0rc48__py3-none-any.whl

mlrun/alerts/alert.py

@@ -112,7 +112,7 @@ class AlertConfig(ModelObj):
                                complex trigger which is based on a prometheus alert
         :param criteria:       When the alert will be triggered based on the specified number of events within the
                                defined time period.
-        :param reset_policy:   When to clear the alert. May be "manual" for manual reset of the alert, or
+        :param reset_policy:   When to clear the alert. Either "manual" for manual reset of the alert, or
                                "auto" if the criteria contains a time period
         :param notifications:  List of notifications to invoke once the alert is triggered
         :param entities:       Entities that the event relates to. The entity object will contain fields that

mlrun/config.py

@@ -631,6 +631,8 @@ default_config = {
         "parquet_batching_max_events": 10_000,
         "parquet_batching_timeout_secs": timedelta(minutes=1).total_seconds(),
         "tdengine": {
+            "run_directly": True,
+            # timeout and retry are ignored when run_directly is set to True
             "timeout": 10,
             "retries": 1,
         },

mlrun/datastore/storeytargets.py

@@ -109,17 +109,20 @@ class StreamStoreyTarget(storey.StreamTarget):
             raise mlrun.errors.MLRunInvalidArgumentError("StreamTarget requires a path")
 
         _, storage_options = get_url_and_storage_options(uri)
-        endpoint, path = parse_path(uri)
+        _, path = parse_path(uri)
 
         access_key = storage_options.get("v3io_access_key")
-        storage = V3ioDriver(
-            webapi=endpoint or mlrun.mlconf.v3io_api, access_key=access_key
-        )
+
+        if alt_key_name := kwargs.pop("alternative_v3io_access_key", None):
+            if alt_key := mlrun.get_secret_or_env(alt_key_name):
+                access_key = alt_key
+
+        storage = V3ioDriver(access_key=access_key)
 
         if storage_options:
             kwargs["storage"] = storage
         if args:
-            args[0] = endpoint
+            args[0] = path
         if "stream_path" in kwargs:
             kwargs["stream_path"] = path
 
@@ -128,6 +131,7 @@ class StreamStoreyTarget(storey.StreamTarget):
 
 class KafkaStoreyTarget(storey.KafkaTarget):
     def __init__(self, *args, **kwargs):
+        kwargs.pop("alternative_v3io_access_key", None)
         path = kwargs.pop("path")
         attributes = kwargs.pop("attributes", {})
         if path and path.startswith("ds://"):

mlrun/datastore/targets.py

@@ -87,7 +87,7 @@ def generate_target_run_id():
 
 def write_spark_dataframe_with_options(spark_options, df, mode, write_format=None):
     # TODO: Replace with just df.sparkSession when Spark 3.2 support is dropped
-    spark_session = getattr(df, "sparkSession") or df.sql_ctx.sparkSession
+    spark_session = getattr(df, "sparkSession", None) or df.sql_ctx.sparkSession
     non_hadoop_spark_options = spark_session_update_hadoop_options(
         spark_session, spark_options
     )

mlrun/model_monitoring/api.py

@@ -160,7 +160,8 @@ def record_results(
     :param context:                  MLRun context. Note that the context is required generating the model endpoint.
     :param infer_results_df:         DataFrame that will be stored under the model endpoint parquet target. Will be
                                      used for doing the drift analysis. Please make sure that the dataframe includes
-                                     both feature names and label columns.
+                                     both feature names and label columns. If you are recording results for existing
+                                     model endpoint, the endpoint should be a batch endpoint.
     :param sample_set_statistics:    Dictionary of sample set statistics that will be used as a reference data for
                                      the current model endpoint.
     :param monitoring_mode:          If enabled, apply model monitoring features on the provided endpoint id. Enabled
@@ -221,23 +222,32 @@ def record_results(
     )
     logger.debug("Model endpoint", endpoint=model_endpoint)
 
-    timestamp = datetime_now()
     if infer_results_df is not None:
-        # Write the monitoring parquet to the relevant model endpoint context
-        write_monitoring_df(
-            feature_set_uri=model_endpoint.spec.monitoring_feature_set_uri,
-            infer_datetime=timestamp,
-            endpoint_id=model_endpoint.metadata.uid,
-            infer_results_df=infer_results_df,
-        )
+        if (
+            model_endpoint.metadata.endpoint_type
+            != mlrun.common.schemas.model_monitoring.EndpointType.BATCH_EP
+        ):
+            logger.warning(
+                "Inference results can be recorded only for batch endpoints. "
+                "Therefore the current results won't be monitored."
+            )
+        else:
+            timestamp = datetime_now()
+            # Write the monitoring parquet to the relevant model endpoint context
+            write_monitoring_df(
+                feature_set_uri=model_endpoint.spec.monitoring_feature_set_uri,
+                infer_datetime=timestamp,
+                endpoint_id=model_endpoint.metadata.uid,
+                infer_results_df=infer_results_df,
+            )
 
-    # Update the last request time
-    update_model_endpoint_last_request(
-        project=project,
-        model_endpoint=model_endpoint,
-        current_request=timestamp,
-        db=db,
-    )
+            # Update the last request time
+            update_model_endpoint_last_request(
+                project=project,
+                model_endpoint=model_endpoint,
+                current_request=timestamp,
+                db=db,
+            )
 
     return model_endpoint
 

mlrun/model_monitoring/applications/evidently/base.py

@@ -12,12 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
+import posixpath
 import uuid
 import warnings
 from abc import ABC
 
 import pandas as pd
 import semver
+from evidently.ui.storage.local.base import METADATA_PATH, FSLocation
 
 import mlrun.model_monitoring.applications.base as mm_base
 import mlrun.model_monitoring.applications.context as mm_context
@@ -81,12 +84,47 @@ class EvidentlyModelMonitoringApplicationBase(
         # TODO : more then one project (mep -> project)
         if not _HAS_EVIDENTLY:
             raise ModuleNotFoundError("Evidently is not installed - the app cannot run")
+        self._log_location(evidently_workspace_path)
         self.evidently_workspace = Workspace.create(evidently_workspace_path)
         self.evidently_project_id = evidently_project_id
         self.evidently_project = self.evidently_workspace.get_project(
             evidently_project_id
         )
 
+    @staticmethod
+    def _log_location(evidently_workspace_path):
+        # TODO remove function + usage after solving issue ML-9530
+        location = FSLocation(base_path=evidently_workspace_path)
+        location.invalidate_cache("")
+        paths = [p for p in location.listdir("") if location.isdir(p)]
+
+        for path in paths:
+            metadata_path = posixpath.join(path, METADATA_PATH)
+            full_path = posixpath.join(location.path, metadata_path)
+            print(f"evidently json issue, working on path: {full_path}")
+            try:
+                with location.open(metadata_path) as f:
+                    content = json.load(f)
+                    print(
+                        f"evidently json issue, successful load path: {full_path}, content: {content}"
+                    )
+            except FileNotFoundError:
+                print(f"evidently json issue, path not found: {full_path}")
+                continue
+            except json.decoder.JSONDecodeError as json_error:
+                print(
+                    f"evidently json issue, path got json error, path:{full_path}, error: {json_error}"
+                )
+                print("evidently json issue, file content:")
+                with location.open(metadata_path) as f:
+                    print(f.read())
+                continue
+            except Exception as error:
+                print(
+                    f"evidently json issue, path got general error, path:{full_path}, error: {error}"
+                )
+                continue
+
     @staticmethod
     def log_evidently_object(
         monitoring_context: mm_context.MonitoringApplicationContext,

mlrun/model_monitoring/controller.py

@@ -12,15 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
 import concurrent.futures
 import datetime
 import json
 import os
 import traceback
+from collections import OrderedDict
 from collections.abc import Iterator
 from contextlib import AbstractContextManager
 from types import TracebackType
-from typing import Any, NamedTuple, Optional, cast
+from typing import Any, NamedTuple, Optional, Union, cast
 
 import nuclio_sdk
 
@@ -30,6 +32,7 @@ import mlrun.feature_store as fstore
 import mlrun.model_monitoring
 import mlrun.model_monitoring.db._schedules as schedules
 import mlrun.model_monitoring.helpers
+import mlrun.platforms.iguazio
 from mlrun.common.schemas import EndpointType
 from mlrun.common.schemas.model_monitoring.constants import (
     ControllerEvent,
@@ -243,7 +246,7 @@ class MonitoringApplicationController:
     Note that the MonitoringApplicationController object requires access keys along with valid project configurations.
     """
 
-    _MAX_OPEN_WINDOWS_ALLOWED = 5
+    _MAX_FEATURE_SET_PER_WORKER = 1000
 
     def __init__(self) -> None:
         """Initialize Monitoring Application Controller"""
@@ -259,6 +262,63 @@ class MonitoringApplicationController:
             mlrun.mlconf.artifact_path
         )
         self.storage_options = store.get_storage_options()
+        self._controller_stream: Optional[
+            Union[
+                mlrun.platforms.iguazio.OutputStream,
+                mlrun.platforms.iguazio.KafkaOutputStream,
+            ]
+        ] = None
+        self._model_monitoring_stream: Optional[
+            Union[
+                mlrun.platforms.iguazio.OutputStream,
+                mlrun.platforms.iguazio.KafkaOutputStream,
+            ]
+        ] = None
+        self.applications_streams: dict[
+            str,
+            Union[
+                mlrun.platforms.iguazio.OutputStream,
+                mlrun.platforms.iguazio.KafkaOutputStream,
+            ],
+        ] = {}
+        self.feature_sets: OrderedDict[str, mlrun.feature_store.FeatureSet] = (
+            collections.OrderedDict()
+        )
+        self.tsdb_connector = mlrun.model_monitoring.get_tsdb_connector(
+            project=self.project
+        )
+
+    @property
+    def controller_stream(
+        self,
+    ) -> Union[
+        mlrun.platforms.iguazio.OutputStream,
+        mlrun.platforms.iguazio.KafkaOutputStream,
+    ]:
+        if self._controller_stream is None:
+            self._controller_stream = mlrun.model_monitoring.helpers.get_output_stream(
+                project=self.project,
+                function_name=mm_constants.MonitoringFunctionNames.APPLICATION_CONTROLLER,
+                v3io_access_key=self.v3io_access_key,
+            )
+        return self._controller_stream
+
+    @property
+    def model_monitoring_stream(
+        self,
+    ) -> Union[
+        mlrun.platforms.iguazio.OutputStream,
+        mlrun.platforms.iguazio.KafkaOutputStream,
+    ]:
+        if self._model_monitoring_stream is None:
+            self._model_monitoring_stream = (
+                mlrun.model_monitoring.helpers.get_output_stream(
+                    project=self.project,
+                    function_name=mm_constants.MonitoringFunctionNames.STREAM,
+                    v3io_access_key=self.model_monitoring_access_key,
+                )
+            )
+        return self._model_monitoring_stream
 
     @staticmethod
     def _get_model_monitoring_access_key() -> Optional[str]:
@@ -422,9 +482,9 @@ class MonitoringApplicationController:
             ]
 
             not_batch_endpoint = (
-                event[ControllerEvent.ENDPOINT_POLICY] != EndpointType.BATCH_EP
+                event[ControllerEvent.ENDPOINT_TYPE] != EndpointType.BATCH_EP
             )
-            m_fs = fstore.get_feature_set(event[ControllerEvent.FEATURE_SET_URI])
+
             logger.info(
                 "Starting analyzing for", timestamp=event[ControllerEvent.TIMESTAMP]
             )
@@ -449,13 +509,39 @@ class MonitoringApplicationController:
                         first_request=first_request,
                         last_request=last_stream_timestamp,
                     ):
-                        df = m_fs.to_dataframe(
-                            start_time=start_infer_time,
-                            end_time=end_infer_time,
-                            time_column=mm_constants.EventFieldType.TIMESTAMP,
-                            storage_options=self.storage_options,
-                        )
-                        if len(df) == 0:
+                        data_in_window = False
+                        if not_batch_endpoint:
+                            # Serving endpoint - get the relevant window data from the TSDB
+                            prediction_metric = self.tsdb_connector.read_predictions(
+                                start=start_infer_time,
+                                end=end_infer_time,
+                                endpoint_id=endpoint_id,
+                            )
+                            if prediction_metric.data:
+                                data_in_window = True
+                        else:
+                            if endpoint_id not in self.feature_sets:
+                                self.feature_sets[endpoint_id] = fstore.get_feature_set(
+                                    event[ControllerEvent.FEATURE_SET_URI]
+                                )
+                            self.feature_sets.move_to_end(endpoint_id, last=False)
+                            if (
+                                len(self.feature_sets)
+                                > self._MAX_FEATURE_SET_PER_WORKER
+                            ):
+                                self.feature_sets.popitem(last=True)
+                            m_fs = self.feature_sets.get(endpoint_id)
+
+                            # Batch endpoint - get the relevant window data from the parquet target
+                            df = m_fs.to_dataframe(
+                                start_time=start_infer_time,
+                                end_time=end_infer_time,
+                                time_column=mm_constants.EventFieldType.TIMESTAMP,
+                                storage_options=self.storage_options,
+                            )
+                            if len(df) > 0:
+                                data_in_window = True
+                        if not data_in_window:
                             logger.info(
                                 "No data found for the given interval",
                                 start=start_infer_time,
@@ -528,8 +614,8 @@ class MonitoringApplicationController:
                 endpoint_id=event[ControllerEvent.ENDPOINT_ID],
             )
 
-    @staticmethod
     def _push_to_applications(
+        self,
         start_infer_time: datetime.datetime,
         end_infer_time: datetime.datetime,
         endpoint_id: str,
@@ -563,12 +649,15 @@ class MonitoringApplicationController:
         }
         for app_name in applications_names:
             data.update({mm_constants.ApplicationEvent.APPLICATION_NAME: app_name})
-
-            app_stream = mlrun.model_monitoring.helpers.get_output_stream(
-                project=project,
-                function_name=app_name,
-                v3io_access_key=model_monitoring_access_key,
-            )
+            if app_name not in self.applications_streams:
+                self.applications_streams[app_name] = (
+                    mlrun.model_monitoring.helpers.get_output_stream(
+                        project=project,
+                        function_name=app_name,
+                        v3io_access_key=model_monitoring_access_key,
+                    )
+                )
+            app_stream = self.applications_streams.get(app_name)
 
             logger.info(
                 "Pushing data to application stream",
@@ -581,7 +670,6 @@ class MonitoringApplicationController:
     def push_regular_event_to_controller_stream(self) -> None:
         """
         pushes a regular event to the controller stream.
-        :param event: the nuclio trigger event
         """
         logger.info("Starting monitoring controller chief")
         applications_names = []
@@ -637,7 +725,6 @@ class MonitoringApplicationController:
                         endpoint,
                         policy,
                         set(applications_names),
-                        self.v3io_access_key,
                         schedule_file,
                     ): endpoint
                     for endpoint in endpoints
@@ -662,7 +749,6 @@ class MonitoringApplicationController:
         endpoint: mlrun.common.schemas.ModelEndpoint,
         policy: dict,
         applications_names: set,
-        v3io_access_key: str,
         schedule_file: schedules.ModelMonitoringSchedulesFileChief,
     ) -> None:
         if self._should_monitor_endpoint(
@@ -688,12 +774,11 @@ class MonitoringApplicationController:
             policy[ControllerEventEndpointPolicy.ENDPOINT_UPDATED] = (
                 endpoint.metadata.updated.isoformat()
             )
-            MonitoringApplicationController.push_to_controller_stream(
+            self.push_to_controller_stream(
                 kind=mm_constants.ControllerEventKind.REGULAR_EVENT,
                 project=endpoint.metadata.project,
                 endpoint_id=endpoint.metadata.uid,
                 endpoint_name=endpoint.metadata.name,
-                stream_access_key=v3io_access_key,
                 timestamp=endpoint.status.last_request.isoformat(
                     sep=" ", timespec="microseconds"
                 ),
@@ -705,13 +790,12 @@ class MonitoringApplicationController:
                 endpoint_policy=policy,
             )
 
-    @staticmethod
     def push_to_controller_stream(
+        self,
         kind: str,
         project: str,
         endpoint_id: str,
         endpoint_name: str,
-        stream_access_key: str,
         timestamp: str,
         first_request: str,
         endpoint_type: int,
@@ -729,7 +813,6 @@ class MonitoringApplicationController:
         :param endpoint_name: the endpoint name string
         :param endpoint_type: Enum of the endpoint type
         :param feature_set_uri: the feature set uri string
-        :param stream_access_key: access key to apply the model monitoring process.
         """
         event = {
             ControllerEvent.KIND.value: kind,
@@ -742,18 +825,13 @@ class MonitoringApplicationController:
             ControllerEvent.FEATURE_SET_URI.value: feature_set_uri,
             ControllerEvent.ENDPOINT_POLICY.value: endpoint_policy,
         }
-        controller_stream = mlrun.model_monitoring.helpers.get_output_stream(
-            project=project,
-            function_name=mm_constants.MonitoringFunctionNames.APPLICATION_CONTROLLER,
-            v3io_access_key=stream_access_key,
-        )
         logger.info(
             "Pushing data to controller stream",
             event=event,
             endpoint_id=endpoint_id,
-            controller_stream_type=str(type(controller_stream)),
+            controller_stream_type=str(type(self.controller_stream)),
         )
-        controller_stream.push([event], partition_key=endpoint_id)
+        self.controller_stream.push([event], partition_key=endpoint_id)
 
     def _push_to_main_stream(self, event: dict, endpoint_id: str) -> None:
         """
@@ -761,18 +839,13 @@ class MonitoringApplicationController:
         :param event: event dictionary to push to stream
         :param endpoint_id: endpoint id string
         """
-        mm_stream = mlrun.model_monitoring.helpers.get_output_stream(
-            project=event.get(ControllerEvent.PROJECT),
-            function_name=mm_constants.MonitoringFunctionNames.APPLICATION_CONTROLLER,
-            v3io_access_key=self.v3io_access_key,
-        )
         logger.info(
             "Pushing data to main stream, NOP event is been generated",
             event=json.dumps(event),
             endpoint_id=endpoint_id,
-            mm_stream_type=str(type(mm_stream)),
+            mm_stream_type=str(type(self.model_monitoring_stream)),
         )
-        mm_stream.push([event], partition_key=endpoint_id)
+        self.model_monitoring_stream.push([event], partition_key=endpoint_id)
 
 
 def handler(context: nuclio_sdk.Context, event: nuclio_sdk.Event) -> None:

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -55,6 +55,9 @@ class TDEngineConnector(TSDBConnector):
 
         self._init_super_tables()
 
+        self._run_directly = (
+            mlrun.mlconf.model_endpoint_monitoring.tdengine.run_directly
+        )
         self._timeout = mlrun.mlconf.model_endpoint_monitoring.tdengine.timeout
         self._retries = mlrun.mlconf.model_endpoint_monitoring.tdengine.retries
 
@@ -74,7 +77,9 @@ class TDEngineConnector(TSDBConnector):
     def _create_connection(self) -> TDEngineConnection:
         """Establish a connection to the TSDB server."""
         logger.debug("Creating a new connection to TDEngine", project=self.project)
-        conn = TDEngineConnection(self._tdengine_connection_profile.dsn())
+        conn = TDEngineConnection(
+            self._tdengine_connection_profile.dsn(), run_directly=self._run_directly
+        )
         conn.prefix_statements = [f"USE {self.database}"]
 
         return conn

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -1090,9 +1090,9 @@ class V3IOTSDBConnector(TSDBConnector):
         Fetch basic metrics from V3IO TSDB and add them to MEP objects.
 
         :param model_endpoint_objects: A list of `ModelEndpoint` objects that will
-                                        be filled with the relevant basic metrics.
+                                       be filled with the relevant basic metrics.
         :param project:                The name of the project.
-        :param run_in_threadpool:      Has no effect.
+        :param run_in_threadpool:      A function that runs another function in a thread pool.
 
         :return: A list of `ModelEndpointMonitoringMetric` objects.
         """
@@ -1104,9 +1104,15 @@ class V3IOTSDBConnector(TSDBConnector):
             uids.append(uid)
             model_endpoint_objects_by_uid[uid] = model_endpoint_object
 
-        error_count_res = self.get_error_count(endpoint_ids=uids, get_raw=True)
-        avg_latency_res = self.get_avg_latency(endpoint_ids=uids, get_raw=True)
-        drift_status_res = self.get_drift_status(endpoint_ids=uids, get_raw=True)
+        error_count_res = await run_in_threadpool(
+            self.get_error_count, endpoint_ids=uids, get_raw=True
+        )
+        avg_latency_res = await run_in_threadpool(
+            self.get_avg_latency, endpoint_ids=uids, get_raw=True
+        )
+        drift_status_res = await run_in_threadpool(
+            self.get_drift_status, endpoint_ids=uids, get_raw=True
+        )
 
         def add_metric(
             metric: str,

mlrun/model_monitoring/helpers.py

@@ -432,58 +432,23 @@ def update_model_endpoint_last_request(
     :param current_request: current request time
     :param db:              DB interface.
     """
-    is_batch_endpoint = (
-        model_endpoint.metadata.endpoint_type == mm_constants.EndpointType.BATCH_EP
-    )
-    if not is_batch_endpoint:
-        logger.info(
-            "Update model endpoint last request time (EP with serving)",
-            project=project,
-            endpoint_id=model_endpoint.metadata.uid,
-            name=model_endpoint.metadata.name,
-            function_name=model_endpoint.spec.function_name,
-            last_request=model_endpoint.status.last_request,
-            current_request=current_request,
-        )
-        db.patch_model_endpoint(
-            project=project,
-            endpoint_id=model_endpoint.metadata.uid,
-            name=model_endpoint.metadata.name,
-            attributes={mm_constants.EventFieldType.LAST_REQUEST: current_request},
-        )
-    else:  # model endpoint without any serving function - close the window "manually"
-        try:
-            time_window = _get_monitoring_time_window_from_controller_run(project, db)
-        except mlrun.errors.MLRunNotFoundError:
-            logger.warn(
-                "Not bumping model endpoint last request time - the monitoring controller isn't deployed yet.\n"
-                "Call `project.enable_model_monitoring()` first."
-            )
-            return
 
-        bumped_last_request = (
-            current_request
-            + time_window
-            + datetime.timedelta(
-                seconds=mlrun.mlconf.model_endpoint_monitoring.parquet_batching_timeout_secs
-            )
-        )
-        logger.info(
-            "Bumping model endpoint last request time (EP without serving)",
-            project=project,
-            endpoint_id=model_endpoint.metadata.uid,
-            function_name=model_endpoint.spec.function_name,
-            last_request=model_endpoint.status.last_request,
-            current_request=current_request.isoformat(),
-            bumped_last_request=bumped_last_request,
-        )
-        db.patch_model_endpoint(
-            project=project,
-            endpoint_id=model_endpoint.metadata.uid,
-            name=model_endpoint.metadata.name,
-            function_name=model_endpoint.spec.function_name,
-            attributes={mm_constants.EventFieldType.LAST_REQUEST: bumped_last_request},
-        )
+    logger.info(
+        "Update model endpoint last request time (EP with serving)",
+        project=project,
+        endpoint_id=model_endpoint.metadata.uid,
+        name=model_endpoint.metadata.name,
+        function_name=model_endpoint.spec.function_name,
+        last_request=model_endpoint.status.last_request,
+        current_request=current_request,
+    )
+    db.patch_model_endpoint(
+        project=project,
+        endpoint_id=model_endpoint.metadata.uid,
+        name=model_endpoint.metadata.name,
+        function_name=model_endpoint.spec.function_name,
+        attributes={mm_constants.EventFieldType.LAST_REQUEST: current_request},
+    )
 
 
 def calculate_inputs_statistics(

mlrun/model_monitoring/stream_processing.py

@@ -264,6 +264,9 @@ class EventStreamProcessor:
                 path=stream_uri,
                 sharding_func=ControllerEvent.ENDPOINT_ID,
                 after="ForwardNOP",
+                # Force using the pipeline key instead of the one in the profile in case of v3io profile.
+                # In case of Kafka, this parameter will be ignored.
+                alternative_v3io_access_key="V3IO_ACCESS_KEY",
             )
 
         apply_push_controller_stream(controller_stream_uri)

mlrun/projects/project.py

@@ -2144,29 +2144,34 @@ class MlrunProject(ModelObj):
         reset_policy: mlrun.common.schemas.alert.ResetPolicy = mlrun.common.schemas.alert.ResetPolicy.AUTO,
     ) -> list[mlrun.alerts.alert.AlertConfig]:
         """
-        :param name:                   The name of the AlertConfig template. It will be combined with mep_id, app-name
-                                       and result name to generate a unique name.
-        :param summary:                Summary of the alert, will be sent in the generated notifications
-        :param endpoints:              The endpoints from which metrics will be retrieved to configure the alerts.
-                                       This `ModelEndpointList` object obtained via the `list_model_endpoints`
-                                       method or created manually using `ModelEndpoint` objects.
-        :param events:                 AlertTrigger event types (EventKind).
-        :param notifications:          List of notifications to invoke once the alert is triggered
-        :param result_names:           Optional. Filters the result names used to create the alert configuration,
-                                       constructed from the app and result_name regex.
-
-                                       For example:
-                                       [`app1.result-*`, `*.result1`]
-                                       will match "mep_uid1.app1.result.result-1" and "mep_uid1.app2.result.result1".
-                                       A specific result_name (not a wildcard) will always create a new alert
-                                       config, regardless of whether the result name exists.
-        :param severity:               Severity of the alert.
-        :param criteria:               When the alert will be triggered based on the
-                                       specified number of events within the defined time period.
-        :param reset_policy:           When to clear the alert. May be "manual" for manual reset of the alert,
-                                       or "auto" if the criteria contains a time period.
-        :returns:                       List of AlertConfig according to endpoints results,
-                                       filtered by result_names.
+        Generate alert configurations based on specified model endpoints and result names, which can be defined
+        explicitly or using regex patterns.
+
+                :param name:                   The name of the AlertConfig template. It will be combined with
+                                               mep id, app name and result name to generate a unique name.
+                :param summary:                Summary of the alert, will be sent in the generated notifications
+                :param endpoints:              The endpoints from which metrics will be retrieved to configure
+                                               the alerts.
+                                               The ModelEndpointList object is obtained via the `list_model_endpoints`
+                                               method or created manually using `ModelEndpoint` objects.
+                :param events:                 AlertTrigger event types (EventKind).
+                :param notifications:          List of notifications to invoke once the alert is triggered
+                :param result_names:           Optional. Filters the result names used to create the alert
+                                               configuration, constructed from the app and result_name regex.
+
+                                               For example:
+                                               [`app1.result-*`, `*.result1`]
+                                               will match "mep_uid1.app1.result.result-1" and
+                                               "mep_uid1.app2.result.result1".
+                                               A specific result_name (not a wildcard) will always create a new alert
+                                               config, regardless of whether the result name exists.
+                :param severity:               Severity of the alert.
+                :param criteria:               The threshold for triggering the alert based on the
+                                               specified number of events within the defined time period.
+                :param reset_policy:           When to clear the alert. Either "manual" for manual reset of the alert,
+                                               or "auto" if the criteria contains a time period.
+                :returns:                      List of AlertConfig according to endpoints results,
+                                               filtered by result_names.
         """
         db = mlrun.db.get_run_db(secrets=self._secrets)
         matching_results = []

mlrun/runtimes/nuclio/function.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import asyncio
+import copy
 import json
 import typing
 import warnings
@@ -50,6 +51,19 @@ from mlrun.runtimes.utils import get_item_name, log_std
 from mlrun.utils import get_in, logger, update_in
 from mlrun_pipelines.common.ops import deploy_op
 
+SENSITIVE_PATHS_IN_TRIGGER_CONFIG = {
+    "password",
+    "secret",
+    "attributes/password",
+    "attributes/accesskeyid",
+    "attributes/secretaccesskey",
+    "attributes/cacert",
+    "attributes/accesskey",
+    "attributes/accesscertificate",
+    "attributes/sasl/password",
+    "attributes/sasl/oauth/clientsecret",
+}
+
 
 def validate_nuclio_version_compatibility(*min_versions):
     """
@@ -274,6 +288,37 @@ class RemoteRuntime(KubeResource):
         if self.metadata.tag:
             mlrun.utils.validate_tag_name(self.metadata.tag, "function.metadata.tag")
 
+    def mask_sensitive_data_in_config(self):
+        if not self.spec.config:
+            return {}
+
+        raw_config = copy.deepcopy(self.spec.config)
+
+        for key, value in self.spec.config.items():
+            if key.startswith("spec.triggers"):
+                trigger_name = key.split(".")[-1]
+
+                for path in SENSITIVE_PATHS_IN_TRIGGER_CONFIG:
+                    # Handle nested keys
+                    nested_keys = path.split("/")
+                    target = value
+                    for sub_key in nested_keys[:-1]:
+                        target = target.get(sub_key, {})
+
+                    last_key = nested_keys[-1]
+                    if last_key in target:
+                        sensitive_field = target[last_key]
+                        if sensitive_field.startswith(
+                            mlrun.model.Credentials.secret_reference_prefix
+                        ):
+                            # already masked
+                            continue
+                        target[last_key] = (
+                            f"{mlrun.model.Credentials.secret_reference_prefix}/spec/triggers/{trigger_name}/{path}"
+                        )
+
+        return raw_config
+
     def set_config(self, key, value):
         self.spec.config[key] = value
         return self
@@ -1230,6 +1275,9 @@ class RemoteRuntime(KubeResource):
             if remote_env.get("name") in credentials_env_var_names:
                 new_env.append(remote_env)
 
+        # update nuclio-specific credentials
+        self.mask_sensitive_data_in_config()
+
         self.spec.env = new_env
 
     def _set_as_mock(self, enable):

mlrun/serving/states.py

@@ -363,15 +363,22 @@ class BaseStep(ModelObj):
                             event: {"x": 5} , result_path="y" means the output of the step will be written
                             to event["y"] resulting in {"x": 5, "y": <result>}
         :param model_endpoint_creation_strategy: Strategy for creating or updating the model endpoint:
-            * **overwrite**:
-            1. If model endpoints with the same name exist, delete the `latest` one.
-            2. Create a new model endpoint entry and set it as `latest`.
-            * **inplace** (default):
-            1. If model endpoints with the same name exist, update the `latest` entry.
-            2. Otherwise, create a new entry.
-            * **archive**:
-            1. If model endpoints with the same name exist, preserve them.
-            2. Create a new model endpoint with the same name and set it to `latest`.
+
+                            * **overwrite**:
+
+                            1. If model endpoints with the same name exist, delete the `latest` one.
+                            2. Create a new model endpoint entry and set it as `latest`.
+
+                            * **inplace** (default):
+
+                            1. If model endpoints with the same name exist, update the `latest` entry.
+                            2. Otherwise, create a new entry.
+
+                            * **archive**:
+
+                            1. If model endpoints with the same name exist, preserve them.
+                            2. Create a new model endpoint with the same name and set it to `latest`.
+
         :param class_args:  class init arguments
         """
         if hasattr(self, "steps"):
@@ -810,15 +817,22 @@ class RouterStep(TaskStep):
         :param handler:    class handler to invoke on run/event
         :param function:   function this step should run in
         :param creation_strategy: Strategy for creating or updating the model endpoint:
-            * **overwrite**:
-            1. If model endpoints with the same name exist, delete the `latest` one.
-            2. Create a new model endpoint entry and set it as `latest`.
-            * **inplace** (default):
-            1. If model endpoints with the same name exist, update the `latest` entry.
-            2. Otherwise, create a new entry.
-            * **archive**:
-            1. If model endpoints with the same name exist, preserve them.
-            2. Create a new model endpoint with the same name and set it to `latest`.
+
+                           * **overwrite**:
+
+                           1. If model endpoints with the same name exist, delete the `latest` one.
+                           2. Create a new model endpoint entry and set it as `latest`.
+
+                           * **inplace** (default):
+
+                           1. If model endpoints with the same name exist, update the `latest` entry.
+                           2. Otherwise, create a new entry.
+
+                           * **archive**:
+
+                           1. If model endpoints with the same name exist, preserve them.
+                           2. Create a new model endpoint with the same name and set it to `latest`.
+
         """
 
         if len(self.routes.keys()) >= MAX_MODELS_PER_ROUTER and key not in self.routes:
@@ -1207,15 +1221,22 @@ class FlowStep(BaseStep):
                             event: {"x": 5} , result_path="y" means the output of the step will be written
                             to event["y"] resulting in {"x": 5, "y": <result>}
         :param model_endpoint_creation_strategy: Strategy for creating or updating the model endpoint:
-            * **overwrite**:
-            1. If model endpoints with the same name exist, delete the `latest` one.
-            2. Create a new model endpoint entry and set it as `latest`.
-            * **inplace** (default):
-            1. If model endpoints with the same name exist, update the `latest` entry.
-            2. Otherwise, create a new entry.
-            * **archive**:
-            1. If model endpoints with the same name exist, preserve them.
-            2. Create a new model endpoint with the same name and set it to `latest`.
+
+                            * **overwrite**:
+
+                            1. If model endpoints with the same name exist, delete the `latest` one.
+                            2. Create a new model endpoint entry and set it as `latest`.
+
+                            * **inplace** (default):
+
+                            1. If model endpoints with the same name exist, update the `latest` entry.
+                            2. Otherwise, create a new entry.
+
+                            * **archive**:
+
+                            1. If model endpoints with the same name exist, preserve them.
+                            2. Create a new model endpoint with the same name and set it to `latest`.
+
         :param class_args:  class init arguments
         """
 

mlrun/utils/helpers.py

@@ -1371,13 +1371,16 @@ def has_timezone(timestamp):
         return False
 
 
-def format_datetime(dt: datetime) -> str:
+def format_datetime(dt: datetime, fmt: Optional[str] = None) -> str:
+    if dt is None:
+        return ""
+
     # If the datetime is naive
     if dt.tzinfo is None:
         dt = dt.replace(tzinfo=timezone.utc)
 
     # TODO: Once Python 3.12 is the minimal version, use %:z to format the timezone offset with a colon
-    formatted_time = dt.strftime("%Y-%m-%d %H:%M:%S.%f%z")
+    formatted_time = dt.strftime(fmt or "%Y-%m-%d %H:%M:%S.%f%z")
 
     # For versions earlier than Python 3.12, we manually insert the colon in the timezone offset
     return formatted_time[:-2] + ":" + formatted_time[-2:]

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "78aff2128b074d4d18751c0cbd2078e14929fa17",
-  "version": "1.8.0-rc46"
+  "git_commit": "c6cc999a67744a5b12083066cb733342de7a8700",
+  "version": "1.8.0-rc48"
 }

{mlrun-1.8.0rc46.dist-info → mlrun-1.8.0rc48.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.8.0rc46
+Version: 1.8.0rc48
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -44,7 +44,7 @@ Requires-Dist: semver~=3.0
 Requires-Dist: dependency-injector~=4.41
 Requires-Dist: fsspec<2024.7,>=2023.9.2
 Requires-Dist: v3iofs~=0.1.17
-Requires-Dist: storey~=1.8.9
+Requires-Dist: storey~=1.8.10
 Requires-Dist: inflection~=0.5.0
 Requires-Dist: python-dotenv~=1.0
 Requires-Dist: setuptools>=75.2
@@ -99,7 +99,7 @@ Requires-Dist: ossfs==2023.12.0; extra == "alibaba-oss"
 Requires-Dist: oss2==2.18.1; extra == "alibaba-oss"
 Provides-Extra: tdengine
 Requires-Dist: taos-ws-py==0.3.2; extra == "tdengine"
-Requires-Dist: taoswswrap~=0.3.2; extra == "tdengine"
+Requires-Dist: taoswswrap~=0.3.4; extra == "tdengine"
 Provides-Extra: snowflake
 Requires-Dist: snowflake-connector-python~=3.7; extra == "snowflake"
 Provides-Extra: kfp18
@@ -152,7 +152,7 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "all"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "all"
 Requires-Dist: sqlalchemy~=1.4; extra == "all"
 Requires-Dist: taos-ws-py==0.3.2; extra == "all"
-Requires-Dist: taoswswrap~=0.3.2; extra == "all"
+Requires-Dist: taoswswrap~=0.3.4; extra == "all"
 Provides-Extra: complete
 Requires-Dist: adlfs==2023.9.0; extra == "complete"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete"
@@ -184,7 +184,7 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "complete"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "complete"
 Requires-Dist: sqlalchemy~=1.4; extra == "complete"
 Requires-Dist: taos-ws-py==0.3.2; extra == "complete"
-Requires-Dist: taoswswrap~=0.3.2; extra == "complete"
+Requires-Dist: taoswswrap~=0.3.4; extra == "complete"
 Provides-Extra: complete-api
 Requires-Dist: adlfs==2023.9.0; extra == "complete-api"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete-api"
@@ -229,7 +229,7 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "complete-api"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "complete-api"
 Requires-Dist: sqlalchemy~=1.4; extra == "complete-api"
 Requires-Dist: taos-ws-py==0.3.2; extra == "complete-api"
-Requires-Dist: taoswswrap~=0.3.2; extra == "complete-api"
+Requires-Dist: taoswswrap~=0.3.4; extra == "complete-api"
 Requires-Dist: timelength~=1.1; extra == "complete-api"
 Requires-Dist: uvicorn~=0.32.1; extra == "complete-api"
 Dynamic: author

{mlrun-1.8.0rc46.dist-info → mlrun-1.8.0rc48.dist-info}/RECORD

@@ -1,6 +1,6 @@
 mlrun/__init__.py,sha256=Cqm9U9eCEdLpMejhU2BEhubu0mHL71igJJIwYa738EA,7450
 mlrun/__main__.py,sha256=0NDzPf9VFRO8KFfGgb8mkGUPIDS285aASV8Hbxs-ND0,45920
-mlrun/config.py,sha256=JEE29XflgA8NsdAOeSmIFBKhS_r-FKHUFwMjcSOYq30,71820
+mlrun/config.py,sha256=GoUHHZ7782V7m6rMYpVVBcKJyoePDBSoEgskDIKVnWY,71931
 mlrun/errors.py,sha256=LkcbXTLANGdsgo2CRX2pdbyNmt--lMsjGv0XZMgP-Nc,8222
 mlrun/execution.py,sha256=FUktsD3puSFjc3LZJU35b-OmFBrBPBNntViCLQVuwnk,50008
 mlrun/features.py,sha256=ReBaNGsBYXqcbgI012n-SO_j6oHIbk_Vpv0CGPXbUmo,15842
@@ -11,7 +11,7 @@ mlrun/render.py,sha256=940H9fBBFeghH4dlifbURvtjlvw4GlWdAXezN6ky4rI,13275
 mlrun/run.py,sha256=n9n5IWBEaOrMIeSakp01DyL09_6FvLy3LCqWpBtvc08,45140
 mlrun/secrets.py,sha256=dZPdkc_zzfscVQepOHUwmzFqnBavDCBXV9DQoH_eIYM,7800
 mlrun/alerts/__init__.py,sha256=0gtG1BG0DXxFrXegIkjbM1XEN4sP9ODo0ucXrNld1hU,601
-mlrun/alerts/alert.py,sha256=9kGTtV385Ax-aTm-450HzPwEek9e0c3O3Qln-jXjhFg,15948
+mlrun/alerts/alert.py,sha256=QQFZGydQbx9RvAaSiaH-ALQZVcDKQX5lgizqj_rXW2k,15948
 mlrun/api/schemas/__init__.py,sha256=tVAnpexDkfI0JWMJNlPSnVOzoV4xqIjWGSln9UkPS4I,13921
 mlrun/artifacts/__init__.py,sha256=ofC2extBCOC1wg1YtdTzWzH3eeG_f-sFBUkHjYtZJpk,1175
 mlrun/artifacts/base.py,sha256=SFHe44o9RV9C3-WODOD53WdBjWk0Ya8lnap9LmERwrQ,29959
@@ -99,8 +99,8 @@ mlrun/datastore/sources.py,sha256=KQp1nNN7TcaewFm3It03H1R28uzlWGZDDHJyqiT--vw,49
 mlrun/datastore/spark_udf.py,sha256=NnnB3DZxZb-rqpRy7b-NC7QWXuuqFn3XkBDc86tU4mQ,1498
 mlrun/datastore/spark_utils.py,sha256=_AsVoU5Ix_-W7Gyq8io8V-2GTk0m8THJNDP3WGGaWJY,2865
 mlrun/datastore/store_resources.py,sha256=PFOMrZ6KH6hBOb0PiO-cHx_kv0UpHu5P2t8_mrR-lS4,6842
-mlrun/datastore/storeytargets.py,sha256=g5zAdizdFkcESoVGxbKWC11ZiXFgM77UL4642G32JaU,6459
-mlrun/datastore/targets.py,sha256=k6IU7XPOYm9jJi5foINfO4NH3NvuXcwYB_B7Rt7V-cg,81195
+mlrun/datastore/storeytargets.py,sha256=dSy9wr4IyxrIE1GHBxzVEeEY1sdU66s4w-oUuaIfa2U,6620
+mlrun/datastore/targets.py,sha256=7qLf26BDH3qYTHOR7TSP0tUMPBhYOkaaOwffUBxgqY0,81201
 mlrun/datastore/utils.py,sha256=CbKbDI6CdFRCqyAXe-jykVvN_GH6R0JkxIQFAogR2GA,10604
 mlrun/datastore/v3io.py,sha256=QSYBORRLcJTeM9mt0EaWzyLcdmzrPkqrF7k5uLTam5U,8209
 mlrun/datastore/vectorstore.py,sha256=k-yom5gfw20hnVG0Rg7aBEehuXwvAloZwn0cx0VGals,11708
@@ -218,11 +218,11 @@ mlrun/launcher/factory.py,sha256=RW7mfzEFi8fR0M-4W1JQg1iq3_muUU6OTqT_3l4Ubrk,233
 mlrun/launcher/local.py,sha256=775HY-8S9LFUX5ubGXrLO0N1lVh8bn-DHFmNYuNqQPA,11451
 mlrun/launcher/remote.py,sha256=rLJW4UAnUT5iUb4BsGBOAV3K4R29a0X4lFtRkVKlyYU,7709
 mlrun/model_monitoring/__init__.py,sha256=ELy7njEtZnz09Dc6PGZSFFEGtnwI15bJNWM3Pj4_YIs,753
-mlrun/model_monitoring/api.py,sha256=Dk5uEAk8HTU00vUwyDPUFqSOgkT9z4gxwaDXbtj-4-U,27737
-mlrun/model_monitoring/controller.py,sha256=mbKonkixNgeXT1BRNKQi6i-bk0pquwWgNclQHAjRvcA,33862
+mlrun/model_monitoring/api.py,sha256=LU58dzE4QZiMH23lgiqfI__3m2E3eEZP-DQe2ioUSwM,28317
+mlrun/model_monitoring/controller.py,sha256=m4Zx_NQ0C-A7WtjBoXnqBmS11RRtLvBaFgbFbIgrdVc,36847
 mlrun/model_monitoring/features_drift_table.py,sha256=c6GpKtpOJbuT1u5uMWDL_S-6N4YPOmlktWMqPme3KFY,25308
-mlrun/model_monitoring/helpers.py,sha256=Cc1TGsqoiuey_XDkwOeaOVV2OnXCryZnBEGGlI8Z524,22620
-mlrun/model_monitoring/stream_processing.py,sha256=4M0H4txMlsC2Q5iKTPp992KWoNPAJjPHj9rqWhXbl8w,33321
+mlrun/model_monitoring/helpers.py,sha256=8QsoYRPOVSnR3Lcv99m4XYrp_cR6hSqBUflYSOkJmFQ,21019
+mlrun/model_monitoring/stream_processing.py,sha256=A66vbrgfWL_sBdAkiPJZmPUFXvQU5phYuyKX6yECtfQ,33558
 mlrun/model_monitoring/tracking_policy.py,sha256=PBIGrUYWrwcE5gwXupBIVzOb0QRRwPJsgQm_yLGQxB4,5595
 mlrun/model_monitoring/writer.py,sha256=ibbhvfSHb8Reqlb7RGFEAUNM4iTyK1gk8-2m46mP6VM,8428
 mlrun/model_monitoring/applications/__init__.py,sha256=xDBxkBjl-whHSG_4t1mLkxiypLH-fzn8TmAW9Mjo2uI,759
@@ -232,7 +232,7 @@ mlrun/model_monitoring/applications/context.py,sha256=DKUDOfN4iY5wpOMjfsarx4pVN9
 mlrun/model_monitoring/applications/histogram_data_drift.py,sha256=09t0tfC35W0SeJA3fzN29pJiB6G-V_8GlcvULVq6H9Q,15179
 mlrun/model_monitoring/applications/results.py,sha256=_qmj6TWT0SR2bi7gUyRKBU418eGgGoLW2_hTJ7S-ock,5782
 mlrun/model_monitoring/applications/evidently/__init__.py,sha256=-DqdPnBSrjZhFvKOu_Ie3MiFvlur9sPTZpZ1u0_1AE8,690
-mlrun/model_monitoring/applications/evidently/base.py,sha256=C8402vQJH7jmY-i49DnYjy6p6dETWex4Tdi8ylFLecA,5097
+mlrun/model_monitoring/applications/evidently/base.py,sha256=_n_2CCQL-fC6hGUZSCLZxZuvXqMqjDHSFX0Giok8HZw,6793
 mlrun/model_monitoring/db/__init__.py,sha256=r47xPGZpIfMuv8J3PQCZTSqVPMhUta4sSJCZFKcS7FM,644
 mlrun/model_monitoring/db/_schedules.py,sha256=RWn4wtKsIXg668gMLpxO9I8GlkxvPSaA5y7w-wFDcgE,9048
 mlrun/model_monitoring/db/_stats.py,sha256=VVMWLMqG3Us3ozBkLaokJF22Ewv8WKmVE1-OvS_g9vA,6943
@@ -242,10 +242,10 @@ mlrun/model_monitoring/db/tsdb/helpers.py,sha256=0oUXc4aUkYtP2SGP6jTb3uPPKImIUsV
 mlrun/model_monitoring/db/tsdb/tdengine/__init__.py,sha256=vgBdsKaXUURKqIf3M0y4sRatmSVA4CQiJs7J5dcVBkQ,620
 mlrun/model_monitoring/db/tsdb/tdengine/schemas.py,sha256=EslhaR65jfeNdD5Ibk-3Hb4e5r5qYPfHb9rTChX3sG0,12689
 mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py,sha256=Uadj0UvAmln2MxDWod-kAzau1uNlqZh981rPhbUH_5M,2857
-mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=gDK6nNbAwTprs2UAI1r7r6loZB40I_8iQ2JvedvAs78,37765
+mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=5jgimfu2-omy8Cnnby7GpgB_MWEp9mmLX0zpbGC2JZ8,37934
 mlrun/model_monitoring/db/tsdb/v3io/__init__.py,sha256=aL3bfmQsUQ-sbvKGdNihFj8gLCK3mSys0qDcXtYOwgc,616
 mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py,sha256=_-zo9relCDtjGgievxAcAP9gVN9nDWs8BzGtFwTjb9M,6284
-mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=GLxg8N1eK7agKgPanOoA1JNbXEH9_kesNRzAPYOgtAQ,46033
+mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=IzdThNwWMBWo0D0VzXV-WVvGg-z7Y9e8ke8_LYJTeVA,46214
 mlrun/model_monitoring/metrics/__init__.py,sha256=6CsTXAxeLbbf8yfCADTaxmiavqwrLEdYFJ-qc5kgDAY,569
 mlrun/model_monitoring/metrics/histogram_distance.py,sha256=E9_WIl2vd6qNvoHVHoFcnuQk3ekbFWOdi8aU7sHrfk4,4724
 mlrun/package/__init__.py,sha256=v7VDyK9kDOOuDvFo4oiGV2fx-vM1KL7fdN9pGLakhUQ,7008
@@ -270,7 +270,7 @@ mlrun/platforms/iguazio.py,sha256=6VBTq8eQ3mzT96tzjYhAtcMQ2VjF4x8LpIPW5DAcX2Q,13
 mlrun/projects/__init__.py,sha256=0Krf0WIKfnZa71WthYOg0SoaTodGg3sV_hK3f_OlTPI,1220
 mlrun/projects/operations.py,sha256=TzPbTYBgmYrjxTKP_wOtBJYFFFwDCQtaVvF1Snr0TfM,20029
 mlrun/projects/pipelines.py,sha256=wud7ezeEmhIJvfYE_wzQbA4ygEfGXHtbOtoOpan6poY,48556
-mlrun/projects/project.py,sha256=Jmr7l9VwFGBItS50DElEN6PuDE6OPPvXn1j16kb1z80,235333
+mlrun/projects/project.py,sha256=WsNZUz_k52llBI5rLBwJeGuIzSlAapVXBQfCL7NVI8E,235765
 mlrun/runtimes/__init__.py,sha256=J9Sy2HiyMlztNv6VUurMzF5H2XzttNil8nRsWDsqLyg,8923
 mlrun/runtimes/base.py,sha256=EL14Kmc1vWEjnBPJwLj5hHC6CtRAQHJLmohCD3sFEHo,37855
 mlrun/runtimes/daskjob.py,sha256=JwuGvOiPsxEDHHMMUS4Oie4hLlYYIZwihAl6DjroTY0,19521
@@ -292,7 +292,7 @@ mlrun/runtimes/mpijob/abstract.py,sha256=JGMjcJ4dvpJbctF6psU9UvYyNCutMxTMgBQeTlz
 mlrun/runtimes/mpijob/v1.py,sha256=1XQZC7AIMGX_AQCbApcwpH8I7y39-v0v2O35MvxjXoo,3213
 mlrun/runtimes/nuclio/__init__.py,sha256=gx1kizzKv8pGT5TNloN1js1hdbxqDw3rM90sLVYVffY,794
 mlrun/runtimes/nuclio/api_gateway.py,sha256=vH9ClKVP4Mb24rvA67xPuAvAhX-gAv6vVtjVxyplhdc,26969
-mlrun/runtimes/nuclio/function.py,sha256=j_gKYhaGfJjr_mVBdUcnSgXcXOHJrKHtUMpmOu8TII8,52979
+mlrun/runtimes/nuclio/function.py,sha256=1EFdGFqlyEfPUVK4Rhh8zWUrff7MNKaHrg7V-bejewg,54618
 mlrun/runtimes/nuclio/nuclio.py,sha256=sLK8KdGO1LbftlL3HqPZlFOFTAAuxJACZCVl1c0Ha6E,2942
 mlrun/runtimes/nuclio/serving.py,sha256=qetAyl-nfn8SWp7KyNgRtMNUVcX_q75SY9dLZP0uH6o,33365
 mlrun/runtimes/nuclio/application/__init__.py,sha256=rRs5vasy_G9IyoTpYIjYDafGoL6ifFBKgBtsXn31Atw,614
@@ -306,7 +306,7 @@ mlrun/serving/remote.py,sha256=gxJkj_J3j-sZcVUbUzbAmJafP6t6y4NVFsu0kWmYngA,18818
 mlrun/serving/routers.py,sha256=SY6AsaiSnh8ssXq8hQE2z9MYapOxFOFJBx9QomiZMO8,53915
 mlrun/serving/server.py,sha256=KiNhW0nTV5STZPzR6kEAUFVzCCAX8qv0g9AoCopARrM,23429
 mlrun/serving/serving_wrapper.py,sha256=R670-S6PX_d5ER6jiHtRvacuPyFzQH0mEf2K0sBIIOM,836
-mlrun/serving/states.py,sha256=Kst2N7R5SaTKYMYB8re9wTlhQwEDgkG61-4JtROKlNI,72803
+mlrun/serving/states.py,sha256=UWiE85MB_SK3rgzWgNqQU2MKeyN2yF2BCvMcMAqLMTs,73247
 mlrun/serving/utils.py,sha256=k2EIYDWHUGkE-IBI6T0UNT32fw-KySsccIJM_LObI00,4171
 mlrun/serving/v1_serving.py,sha256=c6J_MtpE-Tqu00-6r4eJOCO6rUasHDal9W2eBIcrl50,11853
 mlrun/serving/v2_serving.py,sha256=b3C5Utv2_AOPrH_hPi3NarjNbAK3kRoeIfqMU4qNuUo,25362
@@ -321,7 +321,7 @@ mlrun/utils/azure_vault.py,sha256=IEFizrDGDbAaoWwDr1WoA88S_EZ0T--vjYtY-i0cvYQ,34
 mlrun/utils/clones.py,sha256=yXOeuLtgIiKZdmjeKK0Z_vIrH19ds5JuoJaCeDjhwOo,7516
 mlrun/utils/condition_evaluator.py,sha256=-nGfRmZzivn01rHTroiGY4rqEv8T1irMyhzxEei-sKc,1897
 mlrun/utils/db.py,sha256=blQgkWMfFH9lcN4sgJQcPQgEETz2Dl_zwbVA0SslpFg,2186
-mlrun/utils/helpers.py,sha256=OCX51Gn7rrKPpvob_lUnyfs1lFhVHtlzSYKHXAqJJ0Q,74393
+mlrun/utils/helpers.py,sha256=0qKuvXA88Xeu_pbIFE9VQQWonxbAQpkiRSzmsxM8jtk,74465
 mlrun/utils/http.py,sha256=t6FrXQstZm9xVVjxqIGiLzrwZNCR4CSienSOuVgNIcI,8706
 mlrun/utils/logger.py,sha256=RG0m1rx6gfkJ-2C1r_p41MMpPiaDYqaYM2lYHDlNZEU,14767
 mlrun/utils/regex.py,sha256=jbR7IiOp6OO0mg9Fl_cVZCpWb9fL9nTPONCUxCDNWXg,5201
@@ -340,11 +340,11 @@ mlrun/utils/notifications/notification/mail.py,sha256=ZyJ3eqd8simxffQmXzqd3bgbAq
 mlrun/utils/notifications/notification/slack.py,sha256=eQvmctTh6wIG5xVOesLLV9S1-UUCu5UEQ9JIJOor3ts,7183
 mlrun/utils/notifications/notification/webhook.py,sha256=NeyIMSBojjjTJaUHmPbxMByp34GxYkl1-16NqzU27fU,4943
 mlrun/utils/version/__init__.py,sha256=7kkrB7hEZ3cLXoWj1kPoDwo4MaswsI2JVOBpbKgPAgc,614
-mlrun/utils/version/version.json,sha256=BdVuIuX64LHL8ZydQn4fu4jqZorhEvAHUQZ0YBVTWr8,89
+mlrun/utils/version/version.json,sha256=VgW5oJpqfzAo9q08BKtzegYzji4XRJ88Lp0aBY1vEaw,89
 mlrun/utils/version/version.py,sha256=eEW0tqIAkU9Xifxv8Z9_qsYnNhn3YH7NRAfM-pPLt1g,1878
-mlrun-1.8.0rc46.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mlrun-1.8.0rc46.dist-info/METADATA,sha256=-rHQLgCZW8i6RUoHhIzNHEJKGFNvyoI_kIgS59Ajzdk,26008
-mlrun-1.8.0rc46.dist-info/WHEEL,sha256=DK49LOLCYiurdXXOXwGJm6U4DkHkg4lcxjhqwRa0CP4,91
-mlrun-1.8.0rc46.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
-mlrun-1.8.0rc46.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.8.0rc46.dist-info/RECORD,,
+mlrun-1.8.0rc48.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mlrun-1.8.0rc48.dist-info/METADATA,sha256=T7yAGdC4Yb-8_y8KkzMQ3oLLlK1vJNQ9sEbfMlZz7oM,26009
+mlrun-1.8.0rc48.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+mlrun-1.8.0rc48.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
+mlrun-1.8.0rc48.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.8.0rc48.dist-info/RECORD,,

{mlrun-1.8.0rc46.dist-info → mlrun-1.8.0rc48.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.0.2)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any