mlrun 1.7.0rc21__py3-none-any.whl → 1.7.0rc22__py3-none-any.whl

mlrun/alerts/alert.py

@@ -30,6 +30,11 @@ class AlertConfig(ModelObj):
         "reset_policy",
         "state",
     ]
+    _fields_to_serialize = ModelObj._fields_to_serialize + [
+        "entities",
+        "notifications",
+        "trigger",
+    ]
 
     def __init__(
         self,
@@ -71,24 +76,44 @@ class AlertConfig(ModelObj):
         if not self.project or not self.name:
             raise mlrun.errors.MLRunBadRequestError("Project and name must be provided")
 
+    def _serialize_field(
+        self, struct: dict, field_name: str = None, strip: bool = False
+    ):
+        if field_name == "entities":
+            if self.entities:
+                return (
+                    self.entities.dict()
+                    if not isinstance(self.entities, dict)
+                    else self.entities
+                )
+            return None
+        if field_name == "notifications":
+            if self.notifications:
+                return [
+                    notification_data.dict()
+                    if not isinstance(notification_data, dict)
+                    else notification_data
+                    for notification_data in self.notifications
+                ]
+            return None
+        if field_name == "trigger":
+            if self.trigger:
+                return (
+                    self.trigger.dict()
+                    if not isinstance(self.trigger, dict)
+                    else self.trigger
+                )
+            return None
+        return super()._serialize_field(struct, field_name, strip)
+
     def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
-        data = super().to_dict(self._dict_fields)
-
-        data["entities"] = (
-            self.entities.dict()
-            if not isinstance(self.entities, dict)
-            else self.entities
-        )
-        data["notifications"] = [
-            notification_data.dict()
-            if not isinstance(notification_data, dict)
-            else notification_data
-            for notification_data in self.notifications
-        ]
-        data["trigger"] = (
-            self.trigger.dict() if not isinstance(self.trigger, dict) else self.trigger
-        )
-        return data
+        if self.entities is None:
+            raise mlrun.errors.MLRunBadRequestError("Alert entity field is missing")
+        if not self.notifications:
+            raise mlrun.errors.MLRunBadRequestError(
+                "Alert must have at least one notification"
+            )
+        return super().to_dict(self._dict_fields)
 
     @classmethod
     def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):

mlrun/config.py

@@ -229,6 +229,9 @@ default_config = {
                     "executing": "24h",
                 }
             },
+            # When the module is reloaded, the maximum depth recursion configuration for the recursive reload
+            # function is used to prevent infinite loop
+            "reload_max_recursion_depth": 100,
         },
         "databricks": {
             "artifact_directory_path": "/mlrun_databricks_runtime/artifacts_dictionaries"

mlrun/execution.py

@@ -111,6 +111,7 @@ class MLClientCtx:
 
         self._project_object = None
         self._allow_empty_resources = None
+        self._reset_on_run = None
 
     def __enter__(self):
         return self
@@ -389,6 +390,7 @@ class MLClientCtx:
             self._state_thresholds = spec.get(
                 "state_thresholds", self._state_thresholds
             )
+            self._reset_on_run = spec.get("reset_on_run", self._reset_on_run)
 
         self._init_dbs(rundb)
 

mlrun/frameworks/__init__.py

@@ -12,5 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+"""
+MLRun provides a quick and easy integration into your code with mlrun.frameworks: a collection of sub-modules
+for the most commonly used machine and deep learning frameworks, providing features such as automatic logging,
+model management, and distributed training.
+"""
+
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 from .parallel_coordinates import compare_db_runs, compare_run_objects

mlrun/launcher/local.py

@@ -69,6 +69,7 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
         notifications: Optional[list[mlrun.model.Notification]] = None,
         returns: Optional[list[Union[str, dict[str, str]]]] = None,
         state_thresholds: Optional[dict[str, int]] = None,
+        reset_on_run: Optional[bool] = None,
     ) -> "mlrun.run.RunObject":
         # do not allow local function to be scheduled
         if self._is_run_local and schedule is not None:
@@ -88,6 +89,7 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
                 name=name,
                 workdir=workdir,
                 handler=handler,
+                reset_on_run=reset_on_run,
             )
 
         # sanity check
@@ -212,6 +214,7 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
         name: Optional[str] = "",
         workdir: Optional[str] = "",
         handler: Optional[str] = None,
+        reset_on_run: Optional[bool] = None,
     ):
         project = project or runtime.metadata.project
         function_name = name or runtime.metadata.name
@@ -250,6 +253,7 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
             fn.spec.build = runtime.spec.build
 
         run.spec.handler = handler
+        run.spec.reset_on_run = reset_on_run
         return fn
 
     @staticmethod

mlrun/launcher/remote.py

@@ -59,6 +59,7 @@ class ClientRemoteLauncher(launcher.ClientBaseLauncher):
         notifications: Optional[list[mlrun.model.Notification]] = None,
         returns: Optional[list[Union[str, dict[str, str]]]] = None,
         state_thresholds: Optional[dict[str, int]] = None,
+        reset_on_run: Optional[bool] = None,
     ) -> "mlrun.run.RunObject":
         self.enrich_runtime(runtime, project)
         run = self._create_run_object(task)

mlrun/model.py

@@ -872,6 +872,7 @@ class RunSpec(ModelObj):
         returns=None,
         notifications=None,
         state_thresholds=None,
+        reset_on_run=None,
     ):
         # A dictionary of parsing configurations that will be read from the inputs the user set. The keys are the inputs
         # keys (parameter names) and the values are the type hint given in the input keys after the colon.
@@ -908,6 +909,7 @@ class RunSpec(ModelObj):
         self.allow_empty_resources = allow_empty_resources
         self._notifications = notifications or []
         self.state_thresholds = state_thresholds or {}
+        self.reset_on_run = reset_on_run
 
     def _serialize_field(
         self, struct: dict, field_name: str = None, strip: bool = False

mlrun/model_monitoring/api.py

@@ -645,6 +645,7 @@ def _create_model_monitoring_function_base(
         app_step = prepare_step.to(class_name=application_class, **application_kwargs)
     else:
         app_step = prepare_step.to(class_name=application_class)
+    app_step.__class__ = mlrun.serving.MonitoringApplicationStep
     app_step.to(
         class_name="mlrun.model_monitoring.applications._application_steps._PushToMonitoringWriter",
         name="PushToMonitoringWriter",

mlrun/model_monitoring/applications/base.py

@@ -21,10 +21,10 @@ import pandas as pd
 import mlrun
 import mlrun.model_monitoring.applications.context as mm_context
 import mlrun.model_monitoring.applications.results as mm_results
-from mlrun.serving.utils import StepToDict
+from mlrun.serving.utils import MonitoringApplicationToDict
 
 
-class ModelMonitoringApplicationBaseV2(StepToDict, ABC):
+class ModelMonitoringApplicationBaseV2(MonitoringApplicationToDict, ABC):
     """
     A base class for a model monitoring application.
     Inherit from this class to create a custom model monitoring application.
@@ -112,7 +112,7 @@ class ModelMonitoringApplicationBaseV2(StepToDict, ABC):
         raise NotImplementedError
 
 
-class ModelMonitoringApplicationBase(StepToDict, ABC):
+class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
     """
     A base class for a model monitoring application.
     Inherit from this class to create a custom model monitoring application.

mlrun/model_monitoring/db/tsdb/base.py

@@ -177,19 +177,6 @@ class TSDBConnector(ABC):
         :return:                   Metric values object or no data object.
         """
 
-    @abstractmethod
-    def read_prediction_metric_for_endpoint_if_exists(
-        self, endpoint_id: str
-    ) -> typing.Optional[mm_schemas.ModelEndpointMonitoringMetric]:
-        """
-        Read the "invocations" metric for the provided model endpoint, and return the metric object
-        if it exists.
-
-        :param endpoint_id: The model endpoint identifier.
-        :return:            `None` if the invocations metric does not exist, otherwise return the
-                            corresponding metric object.
-        """
-
     @staticmethod
     def df_to_metrics_values(
         *,

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -377,21 +377,25 @@ class TDEngineConnector(TSDBConnector):
             ),  # pyright: ignore[reportArgumentType]
         )
 
-    def read_prediction_metric_for_endpoint_if_exists(
-        self, endpoint_id: str
-    ) -> typing.Optional[mm_schemas.ModelEndpointMonitoringMetric]:
-        # Read just one record, because we just want to check if there is any data for this endpoint_id
-        predictions = self.read_predictions(
-            endpoint_id=endpoint_id,
-            start=datetime.min,
-            end=mlrun.utils.now_date(),
-            limit=1,
-        )
-        if predictions:
-            return mm_schemas.ModelEndpointMonitoringMetric(
-                project=self.project,
-                app=mm_schemas.SpecialApps.MLRUN_INFRA,
-                type=mm_schemas.ModelEndpointMonitoringMetricType.METRIC,
-                name=mm_schemas.PredictionsQueryConstants.INVOCATIONS,
-                full_name=get_invocations_fqn(self.project),
-            )
+    # Note: this function serves as a reference for checking the TSDB for the existence of a metric.
+    #
+    # def read_prediction_metric_for_endpoint_if_exists(
+    #     self, endpoint_id: str
+    # ) -> typing.Optional[mm_schemas.ModelEndpointMonitoringMetric]:
+    #     """
+    #     Read the "invocations" metric for the provided model endpoint, and return the metric object
+    #     if it exists.
+    #
+    #     :param endpoint_id: The model endpoint identifier.
+    #     :return:            `None` if the invocations metric does not exist, otherwise return the
+    #                         corresponding metric object.
+    #     """
+    #     # Read just one record, because we just want to check if there is any data for this endpoint_id
+    #     predictions = self.read_predictions(
+    #         endpoint_id=endpoint_id,
+    #         start=datetime.min,
+    #         end=mlrun.utils.now_date(),
+    #         limit=1,
+    #     )
+    #     if predictions:
+    #         return get_invocations_metric(self.project)

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -12,15 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import typing
 from datetime import datetime
 from io import StringIO
 from typing import Literal, Optional, Union
 
 import pandas as pd
+import v3io_frames
 import v3io_frames.client
-import v3io_frames.errors
-from v3io_frames.frames_pb2 import IGNORE
 
 import mlrun.common.model_monitoring
 import mlrun.common.schemas.model_monitoring as mm_schemas
@@ -35,6 +33,14 @@ _TSDB_RATE = "1/s"
 _CONTAINER = "users"
 
 
+def _is_no_schema_error(exc: v3io_frames.ReadError) -> bool:
+    """
+    In case of a nonexistent TSDB table - a `v3io_frames.ReadError` error is raised.
+    Check if the error message contains the relevant string to verify the cause.
+    """
+    return "No TSDB schema file found" in str(exc)
+
+
 class V3IOTSDBConnector(TSDBConnector):
     """
     Handles the TSDB operations when the TSDB connector is of type V3IO. To manage these operations we use V3IO Frames
@@ -47,7 +53,7 @@ class V3IOTSDBConnector(TSDBConnector):
         self,
         project: str,
         container: str = _CONTAINER,
-        v3io_framesd: typing.Optional[str] = None,
+        v3io_framesd: Optional[str] = None,
         create_table: bool = False,
     ) -> None:
         super().__init__(project=project)
@@ -132,7 +138,7 @@ class V3IOTSDBConnector(TSDBConnector):
             self._frames_client.create(
                 backend=_TSDB_BE,
                 table=table,
-                if_exists=IGNORE,
+                if_exists=v3io_frames.IGNORE,
                 rate=_TSDB_RATE,
             )
 
@@ -162,7 +168,7 @@ class V3IOTSDBConnector(TSDBConnector):
             time_col=mm_schemas.EventFieldType.TIMESTAMP,
             container=self.container,
             v3io_frames=self.v3io_framesd,
-            columns=["latency"],
+            columns=[mm_schemas.EventFieldType.LATENCY],
             index_cols=[
                 mm_schemas.EventFieldType.ENDPOINT_ID,
             ],
@@ -280,7 +286,7 @@ class V3IOTSDBConnector(TSDBConnector):
                 index_cols=index_cols,
             )
             logger.info("Updated V3IO TSDB successfully", table=table)
-        except v3io_frames.errors.Error as err:
+        except v3io_frames.Error as err:
             logger.exception(
                 "Could not write drift measures to TSDB",
                 err=err,
@@ -291,7 +297,7 @@ class V3IOTSDBConnector(TSDBConnector):
                 f"Failed to write application result to TSDB: {err}"
             )
 
-    def delete_tsdb_resources(self, table: typing.Optional[str] = None):
+    def delete_tsdb_resources(self, table: Optional[str] = None):
         if table:
             # Delete a specific table
             tables = [table]
@@ -301,7 +307,7 @@ class V3IOTSDBConnector(TSDBConnector):
         for table_to_delete in tables:
             try:
                 self._frames_client.delete(backend=_TSDB_BE, table=table_to_delete)
-            except v3io_frames.errors.DeleteError as e:
+            except v3io_frames.DeleteError as e:
                 logger.warning(
                     f"Failed to delete TSDB table '{table}'",
                     err=mlrun.errors.err_to_str(e),
@@ -362,7 +368,7 @@ class V3IOTSDBConnector(TSDBConnector):
                 ]
                 metrics_mapping[metric] = values
 
-        except v3io_frames.errors.Error as err:
+        except v3io_frames.Error as err:
             logger.warn("Failed to read tsdb", err=err, endpoint=endpoint_id)
 
         return metrics_mapping
@@ -372,12 +378,11 @@ class V3IOTSDBConnector(TSDBConnector):
         table: str,
         start: Union[datetime, str],
         end: Union[datetime, str],
-        columns: typing.Optional[list[str]] = None,
+        columns: Optional[list[str]] = None,
         filter_query: str = "",
-        interval: typing.Optional[str] = None,
-        agg_funcs: typing.Optional[list] = None,
-        limit: typing.Optional[int] = None,
-        sliding_window_step: typing.Optional[str] = None,
+        interval: Optional[str] = None,
+        agg_funcs: Optional[list[str]] = None,
+        sliding_window_step: Optional[str] = None,
         **kwargs,
     ) -> pd.DataFrame:
         """
@@ -400,7 +405,6 @@ class V3IOTSDBConnector(TSDBConnector):
         :param agg_funcs:             The aggregation functions to apply on the columns. Note that if `agg_funcs` is
                                       provided, `interval` must bg provided as well. Provided as a list of strings in
                                       the format of ['sum', 'avg', 'count', ...].
-        :param limit:                 The maximum number of records to return.
         :param sliding_window_step:   The time step for which the time window moves forward. Note that if
                                       `sliding_window_step` is provided, interval must be provided as well. Provided
                                       as a string in the format of '1m', '1h', etc.
@@ -416,7 +420,7 @@ class V3IOTSDBConnector(TSDBConnector):
 
         if agg_funcs:
             # Frames client expects the aggregators to be a comma-separated string
-            agg_funcs = ",".join(agg_funcs)
+            aggregators = ",".join(agg_funcs)
         table_path = self.tables[table]
         try:
             df = self._frames_client.read(
@@ -427,18 +431,16 @@ class V3IOTSDBConnector(TSDBConnector):
                 columns=columns,
                 filter=filter_query,
                 aggregation_window=interval,
-                aggregators=agg_funcs,
+                aggregators=aggregators,
                 step=sliding_window_step,
                 **kwargs,
             )
         except v3io_frames.ReadError as err:
-            if "No TSDB schema file found" in str(err):
+            if _is_no_schema_error(err):
                 return pd.DataFrame()
             else:
                 raise err
 
-        if limit:
-            df = df.head(limit)
         return df
 
     def _get_v3io_source_directory(self) -> str:
@@ -509,8 +511,8 @@ class V3IOTSDBConnector(TSDBConnector):
             raise ValueError(f"Invalid {type = }")
 
         query = self._get_sql_query(
-            endpoint_id,
-            [(metric.app, metric.name) for metric in metrics],
+            endpoint_id=endpoint_id,
+            metric_and_app_names=[(metric.app, metric.name) for metric in metrics],
             table_path=table_path,
             name=name,
         )
@@ -536,21 +538,28 @@ class V3IOTSDBConnector(TSDBConnector):
 
     @staticmethod
     def _get_sql_query(
+        *,
         endpoint_id: str,
-        names: list[tuple[str, str]],
         table_path: str,
         name: str = mm_schemas.ResultData.RESULT_NAME,
+        metric_and_app_names: Optional[list[tuple[str, str]]] = None,
+        columns: Optional[list[str]] = None,
     ) -> str:
         """Get the SQL query for the results/metrics table"""
+        if columns:
+            selection = ",".join(columns)
+        else:
+            selection = "*"
+
         with StringIO() as query:
             query.write(
-                f"SELECT * FROM '{table_path}' "
+                f"SELECT {selection} FROM '{table_path}' "
                 f"WHERE {mm_schemas.WriterEvent.ENDPOINT_ID}='{endpoint_id}'"
             )
-            if names:
+            if metric_and_app_names:
                 query.write(" AND (")
 
-                for i, (app_name, result_name) in enumerate(names):
+                for i, (app_name, result_name) in enumerate(metric_and_app_names):
                     sub_cond = (
                         f"({mm_schemas.WriterEvent.APPLICATION_NAME}='{app_name}' "
                         f"AND {name}='{result_name}')"
@@ -572,7 +581,6 @@ class V3IOTSDBConnector(TSDBConnector):
         end: Union[datetime, str],
         aggregation_window: Optional[str] = None,
         agg_funcs: Optional[list[str]] = None,
-        limit: Optional[int] = None,
     ) -> Union[
         mm_schemas.ModelEndpointMonitoringMetricNoData,
         mm_schemas.ModelEndpointMonitoringMetricValues,
@@ -591,7 +599,6 @@ class V3IOTSDBConnector(TSDBConnector):
             filter_query=f"endpoint_id=='{endpoint_id}'",
             interval=aggregation_window,
             agg_funcs=agg_funcs,
-            limit=limit,
             sliding_window_step=aggregation_window,
         )
 
@@ -619,18 +626,33 @@ class V3IOTSDBConnector(TSDBConnector):
             ),  # pyright: ignore[reportArgumentType]
         )
 
-    def read_prediction_metric_for_endpoint_if_exists(
-        self, endpoint_id: str
-    ) -> Optional[mm_schemas.ModelEndpointMonitoringMetric]:
-        # Read just one record, because we just want to check if there is any data for this endpoint_id
-        predictions = self.read_predictions(
-            endpoint_id=endpoint_id, start="0", end="now", limit=1
-        )
-        if predictions.data:
-            return mm_schemas.ModelEndpointMonitoringMetric(
-                project=self.project,
-                app=mm_schemas.SpecialApps.MLRUN_INFRA,
-                type=mm_schemas.ModelEndpointMonitoringMetricType.METRIC,
-                name=mm_schemas.PredictionsQueryConstants.INVOCATIONS,
-                full_name=get_invocations_fqn(self.project),
-            )
+    # Note: this function serves as a reference for checking the TSDB for the existence of a metric.
+    #
+    # def read_prediction_metric_for_endpoint_if_exists(
+    #     self, endpoint_id: str
+    # ) -> Optional[mm_schemas.ModelEndpointMonitoringMetric]:
+    #     """
+    #     Read the count of the latency column in the predictions table for the given endpoint_id.
+    #     We just want to check if there is any data for this endpoint_id.
+    #     """
+    #     query = self._get_sql_query(
+    #         endpoint_id=endpoint_id,
+    #         table_path=self.tables[mm_schemas.FileTargetKind.PREDICTIONS],
+    #         columns=[f"count({mm_schemas.EventFieldType.LATENCY})"],
+    #     )
+    #     try:
+    #         logger.debug("Checking TSDB", project=self.project, query=query)
+    #         df: pd.DataFrame = self._frames_client.read(
+    #             backend=_TSDB_BE, query=query, start="0", end="now"
+    #         )
+    #     except v3io_frames.ReadError as err:
+    #         if _is_no_schema_error(err):
+    #             logger.debug(
+    #                 "No predictions yet", project=self.project, endpoint_id=endpoint_id
+    #             )
+    #             return
+    #         else:
+    #             raise
+    #
+    #     if not df.empty:
+    #         return get_invocations_metric(self.project)

mlrun/model_monitoring/helpers.py

@@ -25,6 +25,7 @@ from mlrun.common.schemas.model_monitoring import (
     EventFieldType,
 )
 from mlrun.common.schemas.model_monitoring.model_endpoints import (
+    ModelEndpointMonitoringMetric,
     ModelEndpointMonitoringMetricType,
     _compose_full_name,
 )
@@ -305,3 +306,19 @@ def get_invocations_fqn(project: str) -> str:
         name=mm_constants.PredictionsQueryConstants.INVOCATIONS,
         type=ModelEndpointMonitoringMetricType.METRIC,
     )
+
+
+def get_invocations_metric(project: str) -> ModelEndpointMonitoringMetric:
+    """
+    Return the invocations metric of any model endpoint in the given project.
+
+    :param project: The project name.
+    :returns:       The model monitoring metric object.
+    """
+    return ModelEndpointMonitoringMetric(
+        project=project,
+        app=mm_constants.SpecialApps.MLRUN_INFRA,
+        type=ModelEndpointMonitoringMetricType.METRIC,
+        name=mm_constants.PredictionsQueryConstants.INVOCATIONS,
+        full_name=get_invocations_fqn(project),
+    )

mlrun/package/__init__.py

@@ -12,7 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
+"""
+MLRun package enables fully-automated experiment and pipeline tracking and reproducibility, and easy passing of
+python objects between remote jobs, while not requiring any form of editing to the actual function original code.
+Simply set the function code in a project and run it, MLRun takes care of the rest.
+
+MLRun uses packagers: classes that perform 2 tasks:
+
+#. **Parsing inputs** - automatically cast the runtime's inputs (user's input passed to the function via
+the ``inputs`` parameter of the ``run`` method) to the relevant hinted type.  (Does not require handling of data items.)
+#. **Logging outputs** - automatically save, log, and upload the function's returned objects by the provided
+log hints (user's input passed to the function via the ``returns`` parameter of the ``run`` method).
+(Does not require handling of files and artifacts.)
+"""
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
 import functools

mlrun/package/packagers/__init__.py

@@ -12,7 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
+r"""
+MLRun comes with the following list of modules, out of the box. All of the packagers listed here
+use the implementation of :ref:`DefaultPackager <mlrun.package.packagers.default\_packager.DefaultPackager>` and are
+available by default at the start of each run.
+"""
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
+
 from .default_packager import DefaultPackager
 from .numpy_packagers import NumPySupportedFormat

mlrun/projects/operations.py

@@ -77,6 +77,7 @@ def run_function(
     notifications: list[mlrun.model.Notification] = None,
     returns: Optional[list[Union[str, dict[str, str]]]] = None,
     builder_env: Optional[list] = None,
+    reset_on_run: Optional[bool] = None,
 ) -> Union[mlrun.model.RunObject, PipelineNodeWrapper]:
     """Run a local or remote task as part of a local/kubeflow pipeline
 
@@ -167,6 +168,9 @@ def run_function(
                               artifact type can be given there. The artifact key must appear in the dictionary as
                               "key": "the_key".
     :param builder_env:     env vars dict for source archive config/credentials e.g. builder_env={"GIT_TOKEN": token}
+    :param reset_on_run:    When True, function python modules would reload prior to code execution.
+                            This ensures latest code changes are executed. This argument must be used in
+                            conjunction with the local=True argument.
     :return: MLRun RunObject or PipelineNodeWrapper
     """
     engine, function = _get_engine_and_function(function, project_object)
@@ -215,6 +219,7 @@ def run_function(
             schedule=schedule,
             notifications=notifications,
             builder_env=builder_env,
+            reset_on_run=reset_on_run,
         )
         if run_result:
             run_result._notified = False

mlrun/projects/project.py

@@ -3258,6 +3258,7 @@ class MlrunProject(ModelObj):
         notifications: list[mlrun.model.Notification] = None,
         returns: Optional[list[Union[str, dict[str, str]]]] = None,
         builder_env: Optional[dict] = None,
+        reset_on_run: bool = None,
     ) -> typing.Union[mlrun.model.RunObject, PipelineNodeWrapper]:
         """Run a local or remote task as part of a local/kubeflow pipeline
 
@@ -3314,6 +3315,10 @@ class MlrunProject(ModelObj):
                                   artifact type can be given there. The artifact key must appear in the dictionary as
                                   "key": "the_key".
         :param builder_env: env vars dict for source archive config/credentials e.g. builder_env={"GIT_TOKEN": token}
+        :param reset_on_run:    When True, function python modules would reload prior to code execution.
+                                This ensures latest code changes are executed. This argument must be used in
+                                conjunction with the local=True argument.
+
         :return: MLRun RunObject or PipelineNodeWrapper
         """
         return run_function(
@@ -3339,6 +3344,7 @@ class MlrunProject(ModelObj):
             notifications=notifications,
             returns=returns,
             builder_env=builder_env,
+            reset_on_run=reset_on_run,
         )
 
     def build_function(

mlrun/runtimes/base.py

@@ -68,6 +68,7 @@ spec_fields = [
     "disable_auto_mount",
     "allow_empty_resources",
     "clone_target_dir",
+    "reset_on_run",
 ]
 
 
@@ -336,6 +337,7 @@ class BaseRuntime(ModelObj):
         notifications: Optional[list[mlrun.model.Notification]] = None,
         returns: Optional[list[Union[str, dict[str, str]]]] = None,
         state_thresholds: Optional[dict[str, int]] = None,
+        reset_on_run: Optional[bool] = None,
         **launcher_kwargs,
     ) -> RunObject:
         """
@@ -390,6 +392,9 @@ class BaseRuntime(ModelObj):
                 standards and is at least 1 minute (-1 for infinite).
                 If the phase is active for longer than the threshold, the run will be aborted.
                 See mlconf.function.spec.state_thresholds for the state options and default values.
+        :param reset_on_run: When True, function python modules would reload prior to code execution.
+                             This ensures latest code changes are executed. This argument must be used in
+                             conjunction with the local=True argument.
         :return: Run context object (RunObject) with run metadata, results and status
         """
         launcher = mlrun.launcher.factory.LauncherFactory().create_launcher(
@@ -418,6 +423,7 @@ class BaseRuntime(ModelObj):
             notifications=notifications,
             returns=returns,
             state_thresholds=state_thresholds,
+            reset_on_run=reset_on_run,
         )
 
     def _get_db_run(self, task: RunObject = None):

mlrun/runtimes/daskjob.py

@@ -494,6 +494,7 @@ class DaskCluster(KubejobRuntime):
         notifications: Optional[list[mlrun.model.Notification]] = None,
         returns: Optional[list[Union[str, dict[str, str]]]] = None,
         state_thresholds: Optional[dict[str, int]] = None,
+        reset_on_run: Optional[bool] = None,
         **launcher_kwargs,
     ) -> RunObject:
         if state_thresholds:

mlrun/runtimes/databricks_job/databricks_runtime.py

@@ -232,6 +232,7 @@ def run_mlrun_databricks_job(context,task_parameters: dict, **kwargs):
         notifications: Optional[list[mlrun.model.Notification]] = None,
         returns: Optional[list[Union[str, dict[str, str]]]] = None,
         state_thresholds: Optional[dict[str, int]] = None,
+        reset_on_run: Optional[bool] = None,
         **launcher_kwargs,
     ) -> RunObject:
         if local:

mlrun/runtimes/local.py

@@ -391,7 +391,13 @@ def load_module(file_name, handler, context):
     if context:
         class_args = copy(context._parameters.get("_init_args", {}))
 
-    return get_handler_extended(handler, context, class_args, namespaces=module)
+    return get_handler_extended(
+        handler,
+        context,
+        class_args,
+        namespaces=module,
+        reload_modules=context._reset_on_run,
+    )
 
 
 def run_exec(cmd, args, env=None, cwd=None):

mlrun/serving/__init__.py

@@ -22,10 +22,17 @@ __all__ = [
     "RouterStep",
     "QueueStep",
     "ErrorStep",
+    "MonitoringApplicationStep",
 ]
 
 from .routers import ModelRouter, VotingEnsemble  # noqa
 from .server import GraphContext, GraphServer, create_graph_server  # noqa
-from .states import ErrorStep, QueueStep, RouterStep, TaskStep  # noqa
+from .states import (
+    ErrorStep,
+    QueueStep,
+    RouterStep,
+    TaskStep,
+    MonitoringApplicationStep,
+)  # noqa
 from .v1_serving import MLModelServer, new_v1_model_server  # noqa
 from .v2_serving import V2ModelServer  # noqa

mlrun/serving/states.py

@@ -12,7 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__all__ = ["TaskStep", "RouterStep", "RootFlowStep", "ErrorStep"]
+__all__ = [
+    "TaskStep",
+    "RouterStep",
+    "RootFlowStep",
+    "ErrorStep",
+    "MonitoringApplicationStep",
+]
 
 import os
 import pathlib
@@ -55,6 +61,7 @@ class StepKinds:
     choice = "choice"
     root = "root"
     error_step = "error_step"
+    monitoring_application = "monitoring_application"
 
 
 _task_step_fields = [
@@ -485,13 +492,15 @@ class TaskStep(BaseStep):
                 class_args[key] = arg
         class_args.update(extra_kwargs)
 
-        # add common args (name, context, ..) only if target class can accept them
-        argspec = getfullargspec(class_object)
-        for key in ["name", "context", "input_path", "result_path", "full_event"]:
-            if argspec.varkw or key in argspec.args:
-                class_args[key] = getattr(self, key)
-        if argspec.varkw or "graph_step" in argspec.args:
-            class_args["graph_step"] = self
+        if not isinstance(self, MonitoringApplicationStep):
+            # add common args (name, context, ..) only if target class can accept them
+            argspec = getfullargspec(class_object)
+
+            for key in ["name", "context", "input_path", "result_path", "full_event"]:
+                if argspec.varkw or key in argspec.args:
+                    class_args[key] = getattr(self, key)
+            if argspec.varkw or "graph_step" in argspec.args:
+                class_args["graph_step"] = self
         return class_args
 
     def get_step_class_object(self, namespace):
@@ -582,6 +591,39 @@ class TaskStep(BaseStep):
         return event
 
 
+class MonitoringApplicationStep(TaskStep):
+    """monitoring application execution step, runs users class code"""
+
+    kind = "monitoring_application"
+    _default_class = ""
+
+    def __init__(
+        self,
+        class_name: Union[str, type] = None,
+        class_args: dict = None,
+        handler: str = None,
+        name: str = None,
+        after: list = None,
+        full_event: bool = None,
+        function: str = None,
+        responder: bool = None,
+        input_path: str = None,
+        result_path: str = None,
+    ):
+        super().__init__(
+            class_name=class_name,
+            class_args=class_args,
+            handler=handler,
+            name=name,
+            after=after,
+            full_event=full_event,
+            function=function,
+            responder=responder,
+            input_path=input_path,
+            result_path=result_path,
+        )
+
+
 class ErrorStep(TaskStep):
     """error execution step, runs a class or handler"""
 
@@ -1323,6 +1365,7 @@ classes_map = {
     "flow": FlowStep,
     "queue": QueueStep,
     "error_step": ErrorStep,
+    "monitoring_application": MonitoringApplicationStep,
 }
 
 

mlrun/serving/utils.py

@@ -46,6 +46,15 @@ def _update_result_body(result_path, event_body, result):
 class StepToDict:
     """auto serialization of graph steps to a python dictionary"""
 
+    meta_keys = [
+        "context",
+        "name",
+        "input_path",
+        "result_path",
+        "full_event",
+        "kwargs",
+    ]
+
     def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
         """convert the step object to a python dictionary"""
         fields = fields or getattr(self, "_dict_fields", None)
@@ -54,24 +63,16 @@ class StepToDict:
         if exclude:
             fields = [field for field in fields if field not in exclude]
 
-        meta_keys = [
-            "context",
-            "name",
-            "input_path",
-            "result_path",
-            "full_event",
-            "kwargs",
-        ]
         args = {
             key: getattr(self, key)
             for key in fields
-            if getattr(self, key, None) is not None and key not in meta_keys
+            if getattr(self, key, None) is not None and key not in self.meta_keys
         }
         # add storey kwargs or extra kwargs
         if "kwargs" in fields and (hasattr(self, "kwargs") or hasattr(self, "_kwargs")):
             kwargs = getattr(self, "kwargs", {}) or getattr(self, "_kwargs", {})
             for key, value in kwargs.items():
-                if key not in meta_keys:
+                if key not in self.meta_keys:
                     args[key] = value
 
         mod_name = self.__class__.__module__
@@ -80,7 +81,9 @@ class StepToDict:
             class_path = f"{mod_name}.{class_path}"
         struct = {
             "class_name": class_path,
-            "name": self.name or self.__class__.__name__,
+            "name": self.name
+            if hasattr(self, "name") and self.name
+            else self.__class__.__name__,
             "class_args": args,
         }
         if hasattr(self, "_STEP_KIND"):
@@ -94,6 +97,11 @@ class StepToDict:
         return struct
 
 
+class MonitoringApplicationToDict(StepToDict):
+    _STEP_KIND = "monitoring_application"
+    meta_keys = []
+
+
 class RouterToDict(StepToDict):
     _STEP_KIND = "router"
 

mlrun/utils/helpers.py

@@ -26,7 +26,7 @@ import sys
 import typing
 import warnings
 from datetime import datetime, timezone
-from importlib import import_module
+from importlib import import_module, reload
 from os import path
 from types import ModuleType
 from typing import Any, Optional
@@ -1019,16 +1019,35 @@ def create_class(pkg_class: str):
     return class_
 
 
-def create_function(pkg_func: str):
+def create_function(pkg_func: str, reload_modules: bool = False):
     """Create a function from a package.module.function string
 
     :param pkg_func:  full function location,
                       e.g. "sklearn.feature_selection.f_classif"
+    :param reload_modules: reload the function again.
     """
     splits = pkg_func.split(".")
     pkg_module = ".".join(splits[:-1])
     cb_fname = splits[-1]
     pkg_module = __import__(pkg_module, fromlist=[cb_fname])
+
+    if reload_modules:
+        # Even though the function appears in the modules list, we need to reload
+        # the code again because it may have changed
+        try:
+            logger.debug("Reloading module", module=pkg_func)
+            _reload(
+                pkg_module,
+                max_recursion_depth=mlrun.mlconf.function.spec.reload_max_recursion_depth,
+            )
+        except Exception as exc:
+            logger.warning(
+                "Failed to reload module. Not all associated modules can be reloaded, import them manually."
+                "Or, with Jupyter, restart the Python kernel.",
+                module=pkg_func,
+                err=mlrun.errors.err_to_str(exc),
+            )
+
     function_ = getattr(pkg_module, cb_fname)
     return function_
 
@@ -1086,8 +1105,14 @@ def get_class(class_name, namespace=None):
     return class_object
 
 
-def get_function(function, namespace):
-    """return function callable object from function name string"""
+def get_function(function, namespaces, reload_modules: bool = False):
+    """Return function callable object from function name string
+
+    :param function: path to the function ([class_name::]function)
+    :param namespaces: one or list of namespaces/modules to search the function in
+    :param reload_modules: reload the function again
+    :return: function handler (callable)
+    """
     if callable(function):
         return function
 
@@ -1096,12 +1121,12 @@ def get_function(function, namespace):
         if not function.endswith(")"):
             raise ValueError('function expression must start with "(" and end with ")"')
         return eval("lambda event: " + function[1:-1], {}, {})
-    function_object = _search_in_namespaces(function, namespace)
+    function_object = _search_in_namespaces(function, namespaces)
     if function_object is not None:
         return function_object
 
     try:
-        function_object = create_function(function)
+        function_object = create_function(function, reload_modules)
     except (ImportError, ValueError) as exc:
         raise ImportError(
             f"state/function init failed, handler '{function}' not found"
@@ -1110,19 +1135,24 @@ def get_function(function, namespace):
 
 
 def get_handler_extended(
-    handler_path: str, context=None, class_args: dict = None, namespaces=None
+    handler_path: str,
+    context=None,
+    class_args: dict = None,
+    namespaces=None,
+    reload_modules: bool = False,
 ):
-    """get function handler from [class_name::]handler string
+    """Get function handler from [class_name::]handler string
 
     :param handler_path:  path to the function ([class_name::]handler)
     :param context:       MLRun function/job client context
     :param class_args:    optional dict of class init kwargs
     :param namespaces:    one or list of namespaces/modules to search the handler in
+    :param reload_modules: reload the function again
     :return: function handler (callable)
     """
     class_args = class_args or {}
     if "::" not in handler_path:
-        return get_function(handler_path, namespaces)
+        return get_function(handler_path, namespaces, reload_modules)
 
     splitted = handler_path.split("::")
     class_path = splitted[0].strip()
@@ -1628,3 +1658,15 @@ def format_alert_summary(
     result = result.replace("{{name}}", alert.name)
     result = result.replace("{{entity}}", event_data.entity.ids[0])
     return result
+
+
+def _reload(module, max_recursion_depth):
+    """Recursively reload modules."""
+    if max_recursion_depth <= 0:
+        return
+
+    reload(module)
+    for attribute_name in dir(module):
+        attribute = getattr(module, attribute_name)
+        if type(attribute) is ModuleType:
+            _reload(attribute, max_recursion_depth - 1)

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "c5042e0b46022eb7905002060b7ad0979d63be54",
-  "version": "1.7.0-rc21"
+  "git_commit": "0cbcbb9417af7ac58072ad981672602728fee57e",
+  "version": "1.7.0-rc22"
 }

{mlrun-1.7.0rc21.dist-info → mlrun-1.7.0rc22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.7.0rc21
+Version: 1.7.0rc22
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv

{mlrun-1.7.0rc21.dist-info → mlrun-1.7.0rc22.dist-info}/RECORD

@@ -1,17 +1,17 @@
 mlrun/__init__.py,sha256=y08M1JcKXy5-9_5WaI9fn5aV5BxIQ5QkbduJK0OxWbA,7470
 mlrun/__main__.py,sha256=F65N1MUdAn5hO4qFuJ1v5M3XSCLHUKv7C010toZd-P4,45852
-mlrun/config.py,sha256=yduWBs1UV35cZ1fp6QV6g7xfXl1o96i_MBKY-2Nm63I,65286
+mlrun/config.py,sha256=sG_gjNcQ8UPoa9-nTfeT2UMLcZhbezcojs4RMHN4Xvk,65499
 mlrun/errors.py,sha256=53oT_uQliD-CEe7jxJZMFlNOT86zCTYBl802MZYluaE,7395
-mlrun/execution.py,sha256=XsWu5u_WFkF2yEufWmdrfWsNQpPUQUq4D1JuyWQDslk,41605
+mlrun/execution.py,sha256=ZSk61dXqnEmxnUYxaAjtSzobMlD1yF9VaW56KLmaiUs,41717
 mlrun/features.py,sha256=m17K_3l9Jktwb9dOwlHLTAPTlemsWrRF7dJhXUX0iJU,15429
 mlrun/k8s_utils.py,sha256=WdUajadvAhTR7sAMQdwFqKeJMimuTyqm02VdwK1A4xU,7023
 mlrun/lists.py,sha256=3PqBdcajdwhTe1XuFsAaHTuFVM2kjwepf31qqE82apg,8384
-mlrun/model.py,sha256=LJzYrium0sXmhEhqtfUoI88o8DMyUbb_1B8u0i02zdM,72082
+mlrun/model.py,sha256=bX9OrxF32jAcgphDPJoZkQ9Ov39ECLvdUvoAEcknJOk,72150
 mlrun/render.py,sha256=uVI4kk7XqMAxamholZ_stPQ0nPUebij50ZpgAdjDQ6U,13131
 mlrun/run.py,sha256=HpvrKd77umuOKXH245EaXKFTkzXeqG7D37e1IxoDsB8,42682
 mlrun/secrets.py,sha256=ibtCK79u7JVBZF6F0SP1-xXXF5MyrLEUs_TCWiJAnlc,7798
 mlrun/alerts/__init__.py,sha256=0gtG1BG0DXxFrXegIkjbM1XEN4sP9ODo0ucXrNld1hU,601
-mlrun/alerts/alert.py,sha256=czDWPmmMffCB2HagMdHSRbRXRk2zSvDwMO_CtaV0ZQw,5083
+mlrun/alerts/alert.py,sha256=dAXff9Y0UY5gwuUiofs6BngiNoebkVWI503oATihAXM,6040
 mlrun/api/schemas/__init__.py,sha256=fEWH4I8hr5AdRJ7yoW44RlFB6NHkYDxyomP5J6ct1z4,14248
 mlrun/artifacts/__init__.py,sha256=daGrLqltI1nE3ES30nm-tanUnxReRzfyxyaxNRx2zbc,1168
 mlrun/artifacts/base.py,sha256=azVkiHaJq9JNFKlb91R1vwkdR2QEqF-rIn7bQIL6rf0,29148
@@ -119,7 +119,7 @@ mlrun/feature_store/retrieval/job.py,sha256=vm50yAqvaazuTGbCOgN_e1Ax8gh-d-qQN4Eb
 mlrun/feature_store/retrieval/local_merger.py,sha256=jM-8ta44PeNUc1cKMPs-TxrO9t8pXbwu_Tw8MZrLxUY,4513
 mlrun/feature_store/retrieval/spark_merger.py,sha256=I2KKEqSwao1AX1l6QqKRaXExUiry4P4ox-Vpc4AUNCg,11659
 mlrun/feature_store/retrieval/storey_merger.py,sha256=5YM0UPrLjGOobulHkowRO-1LuvFD2cm_0GxcpnTdu0I,6314
-mlrun/frameworks/__init__.py,sha256=qRHe_nUfxpoLaSASAkIxcW6IyunMtxq5LXhjzZMO_1E,743
+mlrun/frameworks/__init__.py,sha256=Qp9pZRiUcU-drO_NlZnr1Z3RkUSjK44ET9drfGGfiFM,1017
 mlrun/frameworks/parallel_coordinates.py,sha256=AJ3TuvffAC4_zN-RVcyTkq1T3lomDqgeNf7hVBmscEw,11517
 mlrun/frameworks/_common/__init__.py,sha256=7afutDCDVp999gyWSWQZMJRKGuW3VP3MFil8cobRsyg,962
 mlrun/frameworks/_common/artifacts_library.py,sha256=f0rtDRQI3BYT2ZvXR4drSXZPYPJG19Sbej-_ru-i0II,8497
@@ -207,16 +207,16 @@ mlrun/launcher/__init__.py,sha256=JL8qkT1lLr1YvW6iP0hmwDTaSR2RfrMDx0-1gWRhTOE,57
 mlrun/launcher/base.py,sha256=ud1qc2v66-84haAVBuQ2e0IsOzvd_bleSVVImwNWhwE,16461
 mlrun/launcher/client.py,sha256=kgju2mvGuVlvJWRk8sL8qTKF0lf_cSPK2nqYz1oZy3E,6196
 mlrun/launcher/factory.py,sha256=RW7mfzEFi8fR0M-4W1JQg1iq3_muUU6OTqT_3l4Ubrk,2338
-mlrun/launcher/local.py,sha256=FOv9-csTduHQFBYi6qJn8iv7BCAY8DatQgK8Uj-Yglw,11094
-mlrun/launcher/remote.py,sha256=BmEvrGoJBsJgS0_uyv3TjV6DvhgONgcKMHCgbjBdmXg,7654
+mlrun/launcher/local.py,sha256=6t-iumKSd5MurNO0jshDAnG8IZWfpGCiW4JtH8UX9qI,11272
+mlrun/launcher/remote.py,sha256=tGICSfWtvUHeR31mbzy6gqHejmDxjPUgjtxXTWhRubg,7699
 mlrun/model_monitoring/__init__.py,sha256=dm5_j0_pwqrdzFwTaEtGnKfv2nVpNaM56nBI-oqLbNU,879
-mlrun/model_monitoring/api.py,sha256=zOYxVK-rTC1ZZkI9ImPb4ifxPk3mzEzQxMGHRSg2AV0,30298
+mlrun/model_monitoring/api.py,sha256=KzPEPTiqsq5ELcV_O5jbPISdS5o9sO9jwWwSvDwG0hE,30363
 mlrun/model_monitoring/application.py,sha256=RJ8HeAPfGO3P2A_dEZYNg60c1wKTADh2YSv8BQ5embg,745
 mlrun/model_monitoring/controller.py,sha256=MQ4BF3vfJSyYZv6HuTuSLt_nqaflgBYyOSwCccbwaio,27981
 mlrun/model_monitoring/controller_handler.py,sha256=J9Y9ppLsQaxyYRl21165Rr7QuI9EM-mk-5veAqs4Bi0,1336
 mlrun/model_monitoring/evidently_application.py,sha256=iOc42IVjj8m6PDBmVcKIMWm46Bu0EdO9SDcH40Eqhyo,769
 mlrun/model_monitoring/features_drift_table.py,sha256=c6GpKtpOJbuT1u5uMWDL_S-6N4YPOmlktWMqPme3KFY,25308
-mlrun/model_monitoring/helpers.py,sha256=I_JVG4BE5Vw8Z6y26_LuicXUrPlJQBzSHksHEoFclps,10887
+mlrun/model_monitoring/helpers.py,sha256=ROGQVZVoX7kgKI17dqIQSH-8r3t0K4oiwMRXPvfUGxE,11479
 mlrun/model_monitoring/model_endpoint.py,sha256=7VX0cBATqLsA4sSinDzouf41ndxqh2mf5bO9BW0G5Z4,4017
 mlrun/model_monitoring/prometheus.py,sha256=cUR4y73GutJB_pA_VCBDl9YtK4PcIJp2wj2rnLVmYi4,7578
 mlrun/model_monitoring/stream_processing.py,sha256=TuFb1W-WYvynxU_kJl9h3xDn5Ir9b1MUwuynak-DK58,42467
@@ -224,7 +224,7 @@ mlrun/model_monitoring/tracking_policy.py,sha256=sQq956akAQpntkrJwIgFWcEq-JpyVcg
 mlrun/model_monitoring/writer.py,sha256=cAQ24HbtWGA8czzaemmjLT4WfDInJ7gPpkkIp9LePBY,10013
 mlrun/model_monitoring/applications/__init__.py,sha256=i793GqYee01mRh_KD6GShvX7UbPBgdJDO4qf9Z3BXEQ,970
 mlrun/model_monitoring/applications/_application_steps.py,sha256=-g9jxIAFM5f22iJaUAQVlM8QRSv6KFT92I4WHmZe_f0,6028
-mlrun/model_monitoring/applications/base.py,sha256=AJ2si33cZdebD3dxbDRIOLtp6t68tRr1SXKq7VMhtVE,11256
+mlrun/model_monitoring/applications/base.py,sha256=buVKyghH4AB3chZ5py1vyMIFnTF-deY8YDf_fPC9BnQ,11307
 mlrun/model_monitoring/applications/context.py,sha256=i-9h6pWyrS8mjw53zd0kb_Dsf9ReS8cSfnth8PvOEI4,8571
 mlrun/model_monitoring/applications/evidently_base.py,sha256=AE_eIz-GEYm3AZTrMCiqF9bcSMlvYk08LJb6bKWAQLg,8057
 mlrun/model_monitoring/applications/histogram_data_drift.py,sha256=HZmNg09SCjAKkIlKmJwqR7hr-8sXrwFEqXgJCitVbXc,13039
@@ -242,23 +242,23 @@ mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py,sha256=yJJZppbKj3PsOANS_
 mlrun/model_monitoring/db/stores/v3io_kv/__init__.py,sha256=6CsTXAxeLbbf8yfCADTaxmiavqwrLEdYFJ-qc5kgDAY,569
 mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py,sha256=o8P80yowPj4NRLonJ8rdSffot1OB-V6i3Ji1m_TWvzs,27399
 mlrun/model_monitoring/db/tsdb/__init__.py,sha256=NR895JSsEvNmINL223GLf8IbJ16b9Wn4XnxobDwivM8,3724
-mlrun/model_monitoring/db/tsdb/base.py,sha256=tS4XbqH_gztqQy58bJ4jO6ZGIXo3mQmpU-jjWH8aJ0k,13658
+mlrun/model_monitoring/db/tsdb/base.py,sha256=l5SNjO3btJSeglOCn4NHnckntAaP6hcZBVRhOty3KSQ,13122
 mlrun/model_monitoring/db/tsdb/helpers.py,sha256=0oUXc4aUkYtP2SGP6jTb3uPPKImIUsVsrb9otX9a7O4,1189
 mlrun/model_monitoring/db/tsdb/tdengine/__init__.py,sha256=vgBdsKaXUURKqIf3M0y4sRatmSVA4CQiJs7J5dcVBkQ,620
 mlrun/model_monitoring/db/tsdb/tdengine/schemas.py,sha256=94u886UtyK40YNtdOX8WiJUImDytygdaqIzFwo_ExzI,8881
 mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py,sha256=x1cWM2ystghHUeDZNgnaN4kI_XjFOnh1FRBRJAX-tsw,1620
-mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=aZ6sRIC5iHXBQlsKGWxgvH1kh7d5lKRymEXAWjkYH-8,15329
+mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=oplt9s-C-OGa__V456nkHwvyBe5YHxcuIJcYV9GFQHY,15521
 mlrun/model_monitoring/db/tsdb/v3io/__init__.py,sha256=aL3bfmQsUQ-sbvKGdNihFj8gLCK3mSys0qDcXtYOwgc,616
 mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py,sha256=qbiyBzrdWLJAKLmJV4K8jUxsAMbKGZ1vip7WNfRcpXM,4764
-mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=7k2WuYMnQbnePzY0fpIvhr5CVlzwfW3J2UZTVUDlJZ8,25409
+mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=L6XXXr4Mkfad3iTwDyfKScdeuEamL-xtTjoKSwefi7w,26194
 mlrun/model_monitoring/metrics/__init__.py,sha256=6CsTXAxeLbbf8yfCADTaxmiavqwrLEdYFJ-qc5kgDAY,569
 mlrun/model_monitoring/metrics/histogram_distance.py,sha256=E9_WIl2vd6qNvoHVHoFcnuQk3ekbFWOdi8aU7sHrfk4,4724
-mlrun/package/__init__.py,sha256=uWILzN42bcq5vFRk6ptxEmn1I5uBWAnhaJr7e4H834w,7082
+mlrun/package/__init__.py,sha256=_h308qe3NijbH2xymZ_G6XIJSGAa3OX24Je76gP855Q,7941
 mlrun/package/context_handler.py,sha256=Z8v7cXAZXa5l3Tgg6IiEVm74Qbp5cOxx30jvkAY3dwo,14589
 mlrun/package/errors.py,sha256=LKF8SSaRIdbkB7JQz6b9U4mZV42Ebnf6ZHu4wKuWqK4,1204
 mlrun/package/packager.py,sha256=xE7U1njB2RXhmiA0kCSmA4i5j84Dd7Bt-H4Fk5OcVLk,15064
 mlrun/package/packagers_manager.py,sha256=g4XuqpKJGrGKYrA38FXZd9gquDv8KUcW1eXA-DesaMA,37161
-mlrun/package/packagers/__init__.py,sha256=rpxpuATMoxCMgHDaVamm0uwocy71e0CSXm85Q5X9tkU,769
+mlrun/package/packagers/__init__.py,sha256=Mq7-KCjDFYPGcDByAROYT9dXHZx9-QDmeqCepqrwFm0,1039
 mlrun/package/packagers/default_packager.py,sha256=QaZyxm03fRTJy5OGBeyVvSpEqnWj3-hSQVbsCjlTpLM,26625
 mlrun/package/packagers/numpy_packagers.py,sha256=k7Vke41LOp1ExbXCKf4FyahBIDlBqSiYrGPMeH0yI7M,25602
 mlrun/package/packagers/pandas_packagers.py,sha256=KPOZj1yiHxV2b1iah4hlwoNQP4JKzt95Fe9Tn9OUPs8,35761
@@ -273,23 +273,23 @@ mlrun/package/utils/type_hint_utils.py,sha256=JYrek6vuN3z7e6MGUD3qBLDfQ03C4puZXN
 mlrun/platforms/__init__.py,sha256=ggSGF7inITs6S-vj9u4S9X_5psgbA0G3GVqf7zu8qYc,2406
 mlrun/platforms/iguazio.py,sha256=1h5BpdAEQJBg2vIt7ySjUADU0ip5OkaMYr0_VREi9ys,13084
 mlrun/projects/__init__.py,sha256=Lv5rfxyXJrw6WGOWJKhBz66M6t3_zsNMCfUD6waPwx4,1153
-mlrun/projects/operations.py,sha256=60ARXslBZdia7NcQvD_OxG0tOm6c8oUMDJkLKjzzMS0,18843
+mlrun/projects/operations.py,sha256=NEN4PmSvLO9QMwSG4TncmBgTKC9wJp7hGo5lA7OYN_Q,19199
 mlrun/projects/pipelines.py,sha256=c9HhMtXk-6kmTCiY-f0Cmd3GWgL_fBFE6HXp2lrhRtE,40009
-mlrun/projects/project.py,sha256=BTk8TmLpCwCuZHHYxQvzpDi4ZI4w9uFIIa2UTqErH7M,180992
+mlrun/projects/project.py,sha256=umxA0OnsZYtVNu2lmXuXCMMA-0s6iNpsa7uv3jT7THM,181355
 mlrun/runtimes/__init__.py,sha256=0-tYDkew-Cr4DM-wztvMbzDA5xq385Jjo-GrtO_84Sc,8741
-mlrun/runtimes/base.py,sha256=j8eVPkpLbI0mN2yRKPn7kOOk7bkM2pAxgoP2AVxIT5M,36940
-mlrun/runtimes/daskjob.py,sha256=xvN8ajs3-_voqxrfz6b3rh_idNw4ypRAkPRWGOnDMGA,19149
+mlrun/runtimes/base.py,sha256=SvaKfWtAKFhrX2RW7wy3B76OUAg9XdTDaJPpbCEhsdY,37323
+mlrun/runtimes/daskjob.py,sha256=_3jQIEroNxG587ZJ0cW5nVJVBb1IcOECor_bkgZHtMk,19194
 mlrun/runtimes/funcdoc.py,sha256=CC9cWRPgBiM2sk4NJTqusjc6O9kZ-49vGA5WRPjREKE,9796
 mlrun/runtimes/function_reference.py,sha256=iWKRe4r2GTc5S8FOIASYUNLwwne8NqIui51PFr8Q4mg,4918
 mlrun/runtimes/generators.py,sha256=v28HdNgxdHvj888G1dTnUeQZz-D9iTO0hoGeZbCdiuQ,7241
 mlrun/runtimes/kubejob.py,sha256=ptBnMTIjukbEznkdixmbGvBqzujXrRzqNfP7ze6M76M,8660
-mlrun/runtimes/local.py,sha256=MKF-8Q3Xy57cskjfC-FjuK8YW5mhG8lde0eklJbOwIQ,21924
+mlrun/runtimes/local.py,sha256=6dc-qDICQOK0iJJZz4KD498YbqyhF-Uk0_iVbaXdC00,22009
 mlrun/runtimes/pod.py,sha256=I9cfZH-u7ZmAHKc8D7htzKILO1K9lzfoHjBOVe29trU,64406
 mlrun/runtimes/remotesparkjob.py,sha256=9DPxDK8x08t9nReMo083TBxJiiqA83mHCbdtxrjj7AU,7426
 mlrun/runtimes/utils.py,sha256=OFATL8d0c5vKN9N2enAu2oS3b4H71RfeG776ZnfZ0J4,14332
 mlrun/runtimes/databricks_job/__init__.py,sha256=kXGBqhLN0rlAx0kTXhozGzFsIdSqW0uTSKMmsLgq_is,569
 mlrun/runtimes/databricks_job/databricks_cancel_task.py,sha256=sIqIg5DQAf4j0wCPA-G0GoxY6vacRddxCy5KDUZszek,2245
-mlrun/runtimes/databricks_job/databricks_runtime.py,sha256=a9W7A8sboteQov0Z9uVcthEU3FGYFf2cdAsi-vdjH1w,12749
+mlrun/runtimes/databricks_job/databricks_runtime.py,sha256=p80j2_jHzlH20dHT-avjfcbaDBTY2re1WjlJjbg5uSQ,12794
 mlrun/runtimes/databricks_job/databricks_wrapper.py,sha256=oJzym54jD957yzxRXiSYpituSV8JV_XJh90YTKIwapY,8684
 mlrun/runtimes/mpijob/__init__.py,sha256=V_1gQD1VHa0Qvjqgyv8RLouH27Sy9YTwj2ZG62o32zU,1049
 mlrun/runtimes/mpijob/abstract.py,sha256=kDWo-IY1FKLZhI30j38Xx9HMhlUvHezfd1DT2ShoxZY,9161
@@ -304,14 +304,14 @@ mlrun/runtimes/nuclio/application/application.py,sha256=f1GwB5IeavDYls1vHeEqkaOT
 mlrun/runtimes/nuclio/application/reverse_proxy.go,sha256=JIIYae6bXzCLf3jXuu49KWPQYoXr_FDQ2Rbo1OWKAd0,3150
 mlrun/runtimes/sparkjob/__init__.py,sha256=_KPvk0qefeLtHO6lxQE_AMOGiMTG_OT48eRCE4Z2ldw,709
 mlrun/runtimes/sparkjob/spark3job.py,sha256=1bNRy72Migrh_ZASQOx7UlSZTbB-xpNc76sz4kfc9UM,41191
-mlrun/serving/__init__.py,sha256=_6HRAOuS2Ehjo3vwx5h1aI_-JppxEAsl4VfEERAbGFE,1078
+mlrun/serving/__init__.py,sha256=-SMRV3q_5cGVPDxRslXPU0zGYZIygs0cSj7WKlOJJUc,1163
 mlrun/serving/merger.py,sha256=PXLn3A21FiLteJHaDSLm5xKNT-80eTTjfHUJnBX1gKY,6116
 mlrun/serving/remote.py,sha256=MrFByphQWmIsKXqw-MOwl2Q1hbtWReYVRKvlcKj9pfw,17980
 mlrun/serving/routers.py,sha256=scvpXD0VmgGRLJb2UqNq0o39ML2_F_SyZ4OXVQhJIOM,55086
 mlrun/serving/server.py,sha256=U27KHG85Q-Eap3bX4sZlutH_YkpTr1oO89MlkHF9ACs,21081
 mlrun/serving/serving_wrapper.py,sha256=R670-S6PX_d5ER6jiHtRvacuPyFzQH0mEf2K0sBIIOM,836
-mlrun/serving/states.py,sha256=RAWaL3f65-WAa86arKXW8G35TA04Gcp_Z5-gPrDVQr8,57395
-mlrun/serving/utils.py,sha256=WO0n_YTO0YVPTjp_90zxRl4vey4flDgw5vaOHK5p_qY,3871
+mlrun/serving/states.py,sha256=n3RPtzwqfQB1o4H80AoVsP5exL3L3i39ONs-CorWGyM,58539
+mlrun/serving/utils.py,sha256=lej7XcUPX1MmHkEOi_0KZRGSpfbmpnE0GK_Sn4zLkHY,4025
 mlrun/serving/v1_serving.py,sha256=by4myxlnwyZ0ijQ5fURilGCK1sUpdQL2Il1VR3Xqpxg,11805
 mlrun/serving/v2_serving.py,sha256=l2J-kWRPf5vH6T-i96WJwmmzRF2QrWAjepmLVnq04rg,23863
 mlrun/track/__init__.py,sha256=LWRUHJt8JyFW17FyNPOVyWd-NXTf1iptzsK9KFj5fuY,765
@@ -325,7 +325,7 @@ mlrun/utils/azure_vault.py,sha256=IEFizrDGDbAaoWwDr1WoA88S_EZ0T--vjYtY-i0cvYQ,34
 mlrun/utils/clones.py,sha256=mJpx4nyFiY6jlBCvFABsNuyi_mr1mvfPWn81vlafpOU,7361
 mlrun/utils/condition_evaluator.py,sha256=-nGfRmZzivn01rHTroiGY4rqEv8T1irMyhzxEei-sKc,1897
 mlrun/utils/db.py,sha256=KEa-vzicUhzIwo1wBXax2ZuXtYgf5to7wnsY3CYCiOQ,1713
-mlrun/utils/helpers.py,sha256=N3E6xhI8l88BYZuexvoFjVflcs8qy3ZO0i_LDXe40K0,53871
+mlrun/utils/helpers.py,sha256=bJdHRqIaLI6th5FK6IA1gKvypoazeYgV0IjtLyKFgRY,55436
 mlrun/utils/http.py,sha256=l_JCPrCq8bfYUcUcAFWUPvb9Xu-93bLGIhV-H-XCU9s,8707
 mlrun/utils/logger.py,sha256=CG5pgkMeU3VAkIP0pSGOwvFtm0tJYzmPVF8jEp2EtlU,9073
 mlrun/utils/regex.py,sha256=b0AUa2THS-ELzJj0grl5b8Stq609F2XomTZkD9SB1fQ,4900
@@ -343,11 +343,11 @@ mlrun/utils/notifications/notification/ipython.py,sha256=ZtVL30B_Ha0VGoo4LxO-voT
 mlrun/utils/notifications/notification/slack.py,sha256=Vc6EHdnVAZe-p4ZWMvLc23YjMIDE3h2flf2b83ATVCA,7286
 mlrun/utils/notifications/notification/webhook.py,sha256=WgfxX1cpm8n2A-O08pwnsP4tzbxxv_vNUSnyXG4uKts,2752
 mlrun/utils/version/__init__.py,sha256=7kkrB7hEZ3cLXoWj1kPoDwo4MaswsI2JVOBpbKgPAgc,614
-mlrun/utils/version/version.json,sha256=BDBFCJsptywyTCem7RVar7VzaoX-twT1aWb1erZPqhk,89
+mlrun/utils/version/version.json,sha256=YTYzIsYgfu2QIFU3klixQlisgpqRt5PED9zmOJONRXA,89
 mlrun/utils/version/version.py,sha256=eEW0tqIAkU9Xifxv8Z9_qsYnNhn3YH7NRAfM-pPLt1g,1878
-mlrun-1.7.0rc21.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mlrun-1.7.0rc21.dist-info/METADATA,sha256=wUlWivi0FCynPb5k0wo2xhLkMgMtgB_G0QJD61CA4ac,19237
-mlrun-1.7.0rc21.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-mlrun-1.7.0rc21.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
-mlrun-1.7.0rc21.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.7.0rc21.dist-info/RECORD,,
+mlrun-1.7.0rc22.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mlrun-1.7.0rc22.dist-info/METADATA,sha256=1YFwgv7LW2rwwHez3LXjihC5KrpjncBo3J5DUb6iDx8,19237
+mlrun-1.7.0rc22.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+mlrun-1.7.0rc22.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
+mlrun-1.7.0rc22.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.7.0rc22.dist-info/RECORD,,