mlrun 1.6.0rc12__py3-none-any.whl → 1.6.0rc13__py3-none-any.whl

mlrun/config.py

@@ -1130,7 +1130,7 @@ class Config:
 
     def is_explicit_ack(self) -> bool:
         return self.httpdb.nuclio.explicit_ack == "enabled" and (
-            not self.nuclio_version or self.nuclio_version >= "1.12.7"
+            not self.nuclio_version or self.nuclio_version >= "1.12.9"
         )
 
 

mlrun/datastore/targets.py

@@ -877,7 +877,7 @@ class ParquetTarget(BaseStoreTarget):
         else:
             storage_options = storage_options or self.storage_options
 
-        graph.add_step(
+        step = graph.add_step(
             name=self.name or "ParquetTarget",
             after=after,
             graph_shape="cylinder",
@@ -894,6 +894,16 @@ class ParquetTarget(BaseStoreTarget):
             **self.attributes,
         )
 
+        original_to_dict = step.to_dict
+
+        def delete_update_last_written(*arg, **kargs):
+            result = original_to_dict(*arg, **kargs)
+            del result["class_args"]["update_last_written"]
+            return result
+
+        # update_last_written is not serializable (ML-5108)
+        step.to_dict = delete_update_last_written
+
     def get_spark_options(self, key_column=None, timestamp_key=None, overwrite=True):
         partition_cols = []
         if timestamp_key:

mlrun/feature_store/retrieval/base.py

@@ -385,19 +385,24 @@ class BaseMerger(abc.ABC):
         pass
 
     def _normalize_timestamp_column(
-        self, entity_timestamp_column, reference_df, featureset_df, featureset_name
+        self,
+        entity_timestamp_column,
+        reference_df,
+        featureset_timestamp,
+        featureset_df,
+        featureset_name,
     ):
         reference_df_timestamp_type = reference_df[entity_timestamp_column].dtype.name
-        featureset_df_timestamp_type = featureset_df[entity_timestamp_column].dtype.name
+        featureset_df_timestamp_type = featureset_df[featureset_timestamp].dtype.name
 
         if reference_df_timestamp_type != featureset_df_timestamp_type:
             logger.info(
                 f"Merger detected timestamp resolution incompatibility between feature set {featureset_name} and "
                 f"others: {reference_df_timestamp_type} and {featureset_df_timestamp_type}. Converting feature set "
-                f"timestamp column '{entity_timestamp_column}' to type {reference_df_timestamp_type}."
+                f"timestamp column '{featureset_timestamp}' to type {reference_df_timestamp_type}."
             )
-            featureset_df[entity_timestamp_column] = featureset_df[
-                entity_timestamp_column
+            featureset_df[featureset_timestamp] = featureset_df[
+                featureset_timestamp
             ].astype(reference_df_timestamp_type)
 
         return featureset_df

mlrun/feature_store/retrieval/dask_merger.py

@@ -53,7 +53,11 @@ class DaskFeatureMerger(BaseMerger):
         from dask.dataframe.multi import merge_asof
 
         featureset_df = self._normalize_timestamp_column(
-            entity_timestamp_column, entity_df, featureset_df, featureset_name
+            entity_timestamp_column,
+            entity_df,
+            featureset_timestamp,
+            featureset_df,
+            featureset_name,
         )
 
         def sort_partition(partition, timestamp):

mlrun/feature_store/retrieval/local_merger.py

@@ -48,7 +48,11 @@ class LocalFeatureMerger(BaseMerger):
         featureset_df.sort_values(by=featureset_timstamp, inplace=True)
 
         featureset_df = self._normalize_timestamp_column(
-            entity_timestamp_column, entity_df, featureset_df, featureset_name
+            entity_timestamp_column,
+            entity_df,
+            featureset_timstamp,
+            featureset_df,
+            featureset_name,
         )
 
         merged_df = pd.merge_asof(

mlrun/model_monitoring/api.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
 import datetime
 import hashlib
@@ -30,6 +29,7 @@ from mlrun.utils import logger
 
 from .batch import VirtualDrift
 from .features_drift_table import FeaturesDriftTablePlot
+from .helpers import bump_model_endpoint_last_request
 from .model_endpoint import ModelEndpoint
 
 # A union of all supported dataset types:
@@ -125,13 +125,14 @@ def record_results(
     model_endpoint_name: str,
     endpoint_id: str = "",
     function_name: str = "",
-    context: mlrun.MLClientCtx = None,
-    infer_results_df: pd.DataFrame = None,
-    sample_set_statistics: typing.Dict[str, typing.Any] = None,
+    context: typing.Optional[mlrun.MLClientCtx] = None,
+    infer_results_df: typing.Optional[pd.DataFrame] = None,
+    sample_set_statistics: typing.Optional[dict[str, typing.Any]] = None,
     monitoring_mode: ModelMonitoringMode = ModelMonitoringMode.enabled,
-    drift_threshold: float = None,
-    possible_drift_threshold: float = None,
+    drift_threshold: typing.Optional[float] = None,
+    possible_drift_threshold: typing.Optional[float] = None,
     trigger_monitoring_job: bool = False,
+    last_in_batch_set: typing.Optional[bool] = True,
     artifacts_tag: str = "",
     default_batch_image="mlrun/mlrun",
 ) -> ModelEndpoint:
@@ -164,6 +165,14 @@ def record_results(
     :param possible_drift_threshold: The threshold of which to mark possible drifts.
     :param trigger_monitoring_job:   If true, run the batch drift job. If not exists, the monitoring batch function
                                      will be registered through MLRun API with the provided image.
+    :param last_in_batch_set:        This flag can (and should only) be used when the model endpoint does not have
+                                     model-monitoring set.
+                                     If set to `True` (the default), this flag marks the current monitoring window
+                                     (on this monitoring endpoint) is completed - the data inferred so far is assumed
+                                     to be the total data for this monitoring window.
+                                     You may want to set this flag to `False` if you want to record multiple results in
+                                     close time proximity ("batch set"). In this case, set this flag to `False` on all
+                                     but the last batch in the set.
     :param artifacts_tag:            Tag to use for all the artifacts resulted from the function. Will be relevant
                                      only if the monitoring batch job has been triggered.
 
@@ -186,6 +195,7 @@ def record_results(
         monitoring_mode=monitoring_mode,
         db_session=db,
     )
+    logger.debug("Model endpoint", endpoint=model_endpoint.to_dict())
 
     if infer_results_df is not None:
         # Write the monitoring parquet to the relevant model endpoint context
@@ -195,6 +205,27 @@ def record_results(
             infer_results_df=infer_results_df,
         )
 
+    if model_endpoint.spec.stream_path == "":
+        if last_in_batch_set:
+            logger.info(
+                "Updating the last request time to mark the current monitoring window as completed",
+                project=project,
+                endpoint_id=model_endpoint.metadata.uid,
+            )
+            bump_model_endpoint_last_request(
+                project=project, model_endpoint=model_endpoint, db=db
+            )
+    else:
+        if last_in_batch_set is not None:
+            logger.warning(
+                "`last_in_batch_set` is not `None`, but the model endpoint has a stream path. "
+                "Ignoring `last_in_batch_set`, as it is relevant only when the model "
+                "endpoint does not have a model monitoring infrastructure in place (i.e. stream path is "
+                " empty). Set `last_in_batch_set` to `None` to resolve this warning.",
+                project=project,
+                endpoint_id=model_endpoint.metadata.uid,
+            )
+
     if trigger_monitoring_job:
         # Run the monitoring batch drift job
         trigger_drift_batch_job(

mlrun/model_monitoring/helpers.py

@@ -11,13 +11,21 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
 
+import datetime
 import typing
 
+import mlrun
 import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas
+from mlrun.common.schemas.model_monitoring import EventFieldType
+from mlrun.errors import MLRunInvalidArgumentError
+from mlrun.model_monitoring.model_endpoint import ModelEndpoint
+from mlrun.utils import logger
+
+if typing.TYPE_CHECKING:
+    from mlrun.db.base import RunDBInterface
 
 
 def get_stream_path(project: str = None, application_name: str = None):
@@ -89,3 +97,53 @@ def get_connection_string(secret_provider: typing.Callable = None) -> str:
         )
         or mlrun.mlconf.model_endpoint_monitoring.endpoint_store_connection
     )
+
+
+def bump_model_endpoint_last_request(
+    project: str,
+    model_endpoint: ModelEndpoint,
+    db: "RunDBInterface",
+    minutes_delta: int = 10,  # TODO: move to config - should be the same as `batch_interval`
+    seconds_delta: int = 1,
+) -> None:
+    """
+    Update the last request field of the model endpoint to be after the current last request time.
+
+    :param project:         Project name.
+    :param model_endpoint:  Model endpoint object.
+    :param db:              DB interface.
+    :param minutes_delta:   Minutes delta to add to the last request time.
+    :param seconds_delta:   Seconds delta to add to the last request time. This is mainly to ensure that the last
+                            request time is strongly greater than the previous one (with respect to the window time)
+                            after adding the minutes delta.
+    """
+    if not model_endpoint.status.last_request:
+        logger.error(
+            "Model endpoint last request time is empty, cannot bump it.",
+            project=project,
+            endpoint_id=model_endpoint.metadata.uid,
+        )
+        raise MLRunInvalidArgumentError("Model endpoint last request time is empty")
+
+    bumped_last_request = (
+        datetime.datetime.fromisoformat(model_endpoint.status.last_request)
+        + datetime.timedelta(
+            minutes=minutes_delta,
+            seconds=seconds_delta,
+        )
+        + datetime.timedelta(
+            seconds=mlrun.mlconf.model_endpoint_monitoring.parquet_batching_timeout_secs
+        )
+    ).strftime(EventFieldType.TIME_FORMAT)
+    logger.info(
+        "Bumping model endpoint last request time",
+        project=project,
+        endpoint_id=model_endpoint.metadata.uid,
+        last_request=model_endpoint.status.last_request,
+        bumped_last_request=bumped_last_request,
+    )
+    db.patch_model_endpoint(
+        project=project,
+        endpoint_id=model_endpoint.metadata.uid,
+        attributes={EventFieldType.LAST_REQUEST: bumped_last_request},
+    )

mlrun/projects/operations.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import warnings
 from typing import Dict, List, Optional, Union
 
 import kfp
@@ -270,10 +271,17 @@ def build_function(
         e.g. extra_args="--skip-tls-verify --build-arg A=val"
     :param force_build: Force building the image, even when no changes were made
     """
+    if not overwrite_build_params:
+        # TODO: change overwrite_build_params default to True in 1.8.0
+        warnings.warn(
+            "The `overwrite_build_params` parameter default will change from 'False' to 'True in 1.8.0.",
+            mlrun.utils.OverwriteBuildParamsWarning,
+        )
+
     engine, function = _get_engine_and_function(function, project_object)
     if function.kind in mlrun.runtimes.RuntimeKinds.nuclio_runtimes():
         raise mlrun.errors.MLRunInvalidArgumentError(
-            "cannot build use deploy_function()"
+            "Cannot build use deploy_function()"
         )
     if engine == "kfp":
         if overwrite_build_params:
@@ -291,15 +299,21 @@ def build_function(
             skip_deployed=skip_deployed,
         )
     else:
-        function.build_config(
-            image=image,
-            base_image=base_image,
-            commands=commands,
-            secret=secret_name,
-            requirements=requirements,
-            overwrite=overwrite_build_params,
-            extra_args=extra_args,
-        )
+        # TODO: remove filter once overwrite_build_params default is changed to True in 1.8.0
+        with warnings.catch_warnings():
+            warnings.simplefilter(
+                "ignore", category=mlrun.utils.OverwriteBuildParamsWarning
+            )
+
+            function.build_config(
+                image=image,
+                base_image=base_image,
+                commands=commands,
+                secret=secret_name,
+                requirements=requirements,
+                overwrite=overwrite_build_params,
+                extra_args=extra_args,
+            )
         ready = function.deploy(
             watch=True,
             with_mlrun=with_mlrun,

mlrun/projects/project.py

@@ -1944,8 +1944,9 @@ class MlrunProject(ModelObj):
         :param default_controller_image: The default image of the model monitoring controller job. Note that the writer
                                          function, which is a real time nuclio functino, will be deployed with the same
                                          image. By default, the image is mlrun/mlrun.
-        :param base_period:              Minutes to determine the frequency in which the model monitoring controller job
-                                         is running. By default, the base period is 5 minutes.
+        :param base_period:              The time period in minutes in which the model monitoring controller job
+                                         runs. By default, the base period is 10 minutes. The schedule for the job
+                                         will be the following cron expression: "*/{base_period} * * * *".
         :return: model monitoring controller job as a dictionary.
         """
         db = mlrun.db.get_run_db(secrets=self._secrets)
@@ -2981,6 +2982,12 @@ class MlrunProject(ModelObj):
         :param extra_args:  A string containing additional builder arguments in the format of command-line options,
             e.g. extra_args="--skip-tls-verify --build-arg A=val"
         """
+        if not overwrite_build_params:
+            # TODO: change overwrite_build_params default to True in 1.8.0
+            warnings.warn(
+                "The `overwrite_build_params` parameter default will change from 'False' to 'True in 1.8.0.",
+                mlrun.utils.OverwriteBuildParamsWarning,
+            )
         default_image_name = mlrun.mlconf.default_project_image_name.format(
             name=self.name
         )
@@ -3052,35 +3059,48 @@ class MlrunProject(ModelObj):
                 FutureWarning,
             )
 
-        self.build_config(
-            image=image,
-            set_as_default=set_as_default,
-            base_image=base_image,
-            commands=commands,
-            secret_name=secret_name,
-            with_mlrun=with_mlrun,
-            requirements=requirements,
-            requirements_file=requirements_file,
-            overwrite_build_params=overwrite_build_params,
-        )
+        if not overwrite_build_params:
+            # TODO: change overwrite_build_params default to True in 1.8.0
+            warnings.warn(
+                "The `overwrite_build_params` parameter default will change from 'False' to 'True in 1.8.0.",
+                mlrun.utils.OverwriteBuildParamsWarning,
+            )
 
-        function = mlrun.new_function("mlrun--project--image--builder", kind="job")
-
-        build = self.spec.build
-        result = self.build_function(
-            function=function,
-            with_mlrun=build.with_mlrun,
-            image=build.image,
-            base_image=build.base_image,
-            commands=build.commands,
-            secret_name=build.secret,
-            requirements=build.requirements,
-            overwrite_build_params=overwrite_build_params,
-            mlrun_version_specifier=mlrun_version_specifier,
-            builder_env=builder_env,
-            extra_args=extra_args,
-            force_build=force_build,
-        )
+        # TODO: remove filter once overwrite_build_params default is changed to True in 1.8.0
+        with warnings.catch_warnings():
+            warnings.simplefilter(
+                "ignore", category=mlrun.utils.OverwriteBuildParamsWarning
+            )
+
+            self.build_config(
+                image=image,
+                set_as_default=set_as_default,
+                base_image=base_image,
+                commands=commands,
+                secret_name=secret_name,
+                with_mlrun=with_mlrun,
+                requirements=requirements,
+                requirements_file=requirements_file,
+                overwrite_build_params=overwrite_build_params,
+            )
+
+            function = mlrun.new_function("mlrun--project--image--builder", kind="job")
+
+            build = self.spec.build
+            result = self.build_function(
+                function=function,
+                with_mlrun=build.with_mlrun,
+                image=build.image,
+                base_image=build.base_image,
+                commands=build.commands,
+                secret_name=build.secret,
+                requirements=build.requirements,
+                overwrite_build_params=overwrite_build_params,
+                mlrun_version_specifier=mlrun_version_specifier,
+                builder_env=builder_env,
+                extra_args=extra_args,
+                force_build=force_build,
+            )
 
         try:
             mlrun.db.get_run_db(secrets=self._secrets).delete_function(

mlrun/runtimes/databricks_job/databricks_runtime.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import os
+from ast import FunctionDef, parse, unparse
 from base64 import b64decode, b64encode
 from typing import Callable, Dict, List, Optional, Union
 
@@ -22,61 +23,92 @@ from mlrun.model import HyperParamOptions, RunObject
 from mlrun.runtimes.kubejob import KubejobRuntime
 
 
+def get_log_artifacts_code(runobj: RunObject, task_parameters: dict):
+    artifact_json_dir = task_parameters.get(
+        "artifact_json_dir",
+        mlrun.mlconf.function.databricks.artifact_directory_path,
+    )
+    artifact_json_path = (
+        f"{artifact_json_dir}/mlrun_artifact_{runobj.metadata.uid}.json"
+    )
+    return (
+        log_artifacts_code_template.format(f"/dbfs{artifact_json_path}"),
+        artifact_json_path,
+    )
+
+
+def replace_log_artifact_function(code: str, log_artifacts_code: str):
+    #  user can use a dummy function in oder to avoid edit his code.
+    #  replace mlrun_log_artifact function if already exist.
+    is_replaced = False
+    parsed_code = parse(code)
+    for node in parsed_code.body:
+        if isinstance(node, FunctionDef) and node.name == "mlrun_log_artifact":
+            new_function_ast = parse(log_artifacts_code)
+            node.args = new_function_ast.body[0].args
+            node.body = new_function_ast.body[0].body
+            is_replaced = True
+            break
+    return unparse(parsed_code), is_replaced
+
+
 class DatabricksRuntime(KubejobRuntime):
     kind = "databricks"
     _is_remote = True
 
-    def _get_log_artifacts_code(self, runobj: RunObject, task_parameters: dict):
-        artifact_json_dir = task_parameters.get(
-            "artifact_json_dir",
-            mlrun.mlconf.function.databricks.artifact_directory_path,
-        )
-        artifact_json_path = (
-            f"{artifact_json_dir}/mlrun_artifact_{runobj.metadata.uid}.json"
+    @staticmethod
+    def _verify_returns(returns):
+        # TODO complete returns feature
+        if returns:
+            raise MLRunInvalidArgumentError(
+                "Databricks function does not support returns."
+            )
+
+    def _get_modified_user_code(self, original_handler: str, log_artifacts_code: str):
+        encoded_code = (
+            self.spec.build.functionSourceCode if hasattr(self.spec, "build") else None
         )
-        return (
-            artifacts_code_template.format(f"/dbfs{artifact_json_path}"),
-            artifact_json_path,
+        if not encoded_code:
+            raise ValueError("Databricks function must be provided with user code")
+
+        decoded_code = b64decode(encoded_code).decode("utf-8")
+        decoded_code, is_replaced = replace_log_artifact_function(
+            code=decoded_code, log_artifacts_code=log_artifacts_code
         )
+        if is_replaced:
+            decoded_code = (
+                logger_and_consts_code + _databricks_script_code + decoded_code
+            )
+        else:
+            decoded_code = (
+                logger_and_consts_code
+                + log_artifacts_code
+                + _databricks_script_code
+                + decoded_code
+            )
+        if original_handler:
+            decoded_code += f"\nresult = {original_handler}(**handler_arguments)\n"
+            decoded_code += _return_artifacts_code
+        return b64encode(decoded_code.encode("utf-8")).decode("utf-8")
 
     def get_internal_parameters(self, runobj: RunObject):
         """
-        Return the internal function code.
+        Return the internal function parameters + code.
         """
         task_parameters = runobj.spec.parameters.get("task_parameters", {})
         if "original_handler" in task_parameters:
             original_handler = task_parameters["original_handler"]
         else:
             original_handler = runobj.spec.handler or ""
-        encoded_code = (
-            self.spec.build.functionSourceCode if hasattr(self.spec, "build") else None
-        )
-        if not encoded_code:
-            raise ValueError("Databricks function must be provided with user code")
-        decoded_code = b64decode(encoded_code).decode("utf-8")
-        artifacts_code, artifact_json_path = self._get_log_artifacts_code(
+        log_artifacts_code, artifact_json_path = get_log_artifacts_code(
             runobj=runobj, task_parameters=task_parameters
         )
-        code = artifacts_code + _databricks_script_code + decoded_code
-        if original_handler:
-            code += f"\nresult = {original_handler}(**handler_arguments)\n"
-            code += """\n
-default_key_template = 'mlrun_return_value_'
-if result:
-    if isinstance(result, dict):
-        for key, path in result.items():
-            mlrun_log_artifact(name=key, path=path)
-    elif isinstance(result, (list, tuple, set)):
-        for index, value in enumerate(result):
-            key = f'{default_key_template}{index+1}'
-            mlrun_log_artifact(name=key, path=value)
-    elif isinstance(result, str):
-        mlrun_log_artifact(name=f'{default_key_template}1', path=result)
-    else:
-        mlrun_logger.warning(f'cannot log artifacts with the result of handler function \
-- result in unsupported type. {type(result)}')
-"""
-        code = b64encode(code.encode("utf-8")).decode("utf-8")
+        returns = runobj.spec.returns or []
+        self._verify_returns(returns=returns)
+        code = self._get_modified_user_code(
+            original_handler=original_handler,
+            log_artifacts_code=log_artifacts_code,
+        )
         updated_task_parameters = {
             "original_handler": original_handler,
             "artifact_json_path": artifact_json_path,
@@ -159,6 +191,22 @@ if result:
         )
 
 
+logger_and_consts_code = """ \n
+import os
+import logging
+mlrun_logger = logging.getLogger('mlrun_logger')
+mlrun_logger.setLevel(logging.DEBUG)
+
+mlrun_console_handler = logging.StreamHandler()
+mlrun_console_handler.setLevel(logging.DEBUG)
+mlrun_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+mlrun_console_handler.setFormatter(mlrun_formatter)
+mlrun_logger.addHandler(mlrun_console_handler)
+
+mlrun_default_artifact_template = 'mlrun_return_value_'
+mlrun_artifact_index = 0
+"""
+
 _databricks_script_code = """
 
 import argparse
@@ -168,16 +216,17 @@ parser.add_argument('handler_arguments')
 handler_arguments = parser.parse_args().handler_arguments
 handler_arguments = json.loads(handler_arguments)
 
-"""
 
-artifacts_code_template = """\n
-import logging
-mlrun_logger = logging.getLogger('mlrun_logger')
-mlrun_logger.setLevel(logging.DEBUG)
+"""
 
-def mlrun_log_artifact(name, path):
-    if not name or not path:
-        mlrun_logger.error(f'name and path required for logging an mlrun artifact - {{name}} : {{path}}')
+log_artifacts_code_template = """\n
+def mlrun_log_artifact(name='', path=''):
+    global mlrun_artifact_index
+    mlrun_artifact_index+=1  #  by how many artifacts we tried to log, not how many succeed.
+    if name is None or name == '':
+        name = f'{{mlrun_default_artifact_template}}{{mlrun_artifact_index}}'
+    if not path:
+        mlrun_logger.error(f'path required for logging an mlrun artifact - {{name}} : {{path}}')
         return
     if not isinstance(name, str) or not isinstance(path, str):
         mlrun_logger.error(f'name and path must be in string type for logging an mlrun artifact - {{name}} : {{path}}')
@@ -186,21 +235,38 @@ def mlrun_log_artifact(name, path):
         mlrun_logger.error(f'path for an mlrun artifact must start with /dbfs or dbfs:/ - {{name}} : {{path}}')
         return
     mlrun_artifacts_path = '{}'
-    import json
-    import os
-    new_data = {{name:path}}
-    if os.path.exists(mlrun_artifacts_path):
-        with open(mlrun_artifacts_path, 'r+') as json_file:
-            existing_data = json.load(json_file)
-            existing_data.update(new_data)
-            json_file.seek(0)
-            json.dump(existing_data, json_file)
-    else:
-        parent_dir = os.path.dirname(mlrun_artifacts_path)
-        if parent_dir != '/dbfs':
-            os.makedirs(parent_dir, exist_ok=True)
-        with open(mlrun_artifacts_path, 'w') as json_file:
-            json.dump(new_data, json_file)
-    mlrun_logger.info(f'successfully wrote artifact details to the artifact JSON file in DBFS - {{name}} : {{path}}')
+    try:
+        new_data = {{name:path}}
+        if os.path.exists(mlrun_artifacts_path):
+            with open(mlrun_artifacts_path, 'r+') as json_file:
+                existing_data = json.load(json_file)
+                existing_data.update(new_data)
+                json_file.seek(0)
+                json.dump(existing_data, json_file)
+        else:
+            parent_dir = os.path.dirname(mlrun_artifacts_path)
+            if parent_dir != '/dbfs':
+                os.makedirs(parent_dir, exist_ok=True)
+            with open(mlrun_artifacts_path, 'w') as json_file:
+                json.dump(new_data, json_file)
+        success_log = f'successfully wrote artifact details to the artifact JSON file in DBFS - {{name}} : {{path}}'
+        mlrun_logger.info(success_log)
+    except Exception as unknown_exception:
+        mlrun_logger.error(f'log mlrun artifact failed - {{name}} : {{path}}. error: {{unknown_exception}}')
 \n
 """
+
+_return_artifacts_code = """\n
+if result:
+    if isinstance(result, dict):
+        for key, path in result.items():
+            mlrun_log_artifact(name=key, path=path)
+    elif isinstance(result, (list, tuple, set)):
+        for artifact_path in result:
+            mlrun_log_artifact(path=artifact_path)
+    elif isinstance(result, str):
+        mlrun_log_artifact(path=result)
+    else:
+        mlrun_logger.warning(f'can not log artifacts with the result of handler function \
+- result in unsupported type. {type(result)}')
+"""

mlrun/runtimes/databricks_job/databricks_wrapper.py

@@ -200,7 +200,6 @@ def run_mlrun_databricks_job(
                 is_finished=True,
             )
         run_output = workspace.jobs.get_run_output(get_task(run).run_id)
-        context.log_result("databricks_runtime_task", run_output.as_dict())
     finally:
         workspace.dbfs.delete(script_path_on_dbfs)
         workspace.dbfs.delete(artifact_json_path)

mlrun/runtimes/function.py

@@ -550,6 +550,13 @@ class RemoteRuntime(KubeResource):
         """
         # todo: verify that the function name is normalized
 
+        old_http_session = getattr(self, "_http_session", None)
+        if old_http_session:
+            # ensure existing http session is terminated prior to (re)deploy to ensure that a connection to an old
+            # replica will not be reused
+            old_http_session.close()
+            self._http_session = None
+
         verbose = verbose or self.verbose
         if verbose:
             self.set_env("MLRUN_LOG_LEVEL", "DEBUG")
@@ -939,7 +946,7 @@ class RemoteRuntime(KubeResource):
                 http_client_kwargs["json"] = body
         try:
             logger.info("invoking function", method=method, path=path)
-            if not hasattr(self, "_http_session"):
+            if not getattr(self, "_http_session", None):
                 self._http_session = requests.Session()
             resp = self._http_session.request(
                 method, path, headers=headers, **http_client_kwargs

mlrun/runtimes/kubejob.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import time
+import warnings
 
 import mlrun.common.schemas
 import mlrun.db
@@ -130,7 +131,12 @@ class KubejobRuntime(KubeResource):
         :param builder_env: Kaniko builder pod env vars dict (for config/credentials)
             e.g. builder_env={"GIT_TOKEN": token}
         """
-
+        if not overwrite:
+            # TODO: change overwrite default to True in 1.8.0
+            warnings.warn(
+                "The `overwrite` parameter default will change from 'False' to 'True in 1.8.0.",
+                mlrun.utils.OverwriteBuildParamsWarning,
+            )
         image = mlrun.utils.helpers.remove_image_protocol_prefix(image)
         self.spec.build.build_config(
             image=image,

mlrun/serving/server.py

@@ -357,7 +357,7 @@ def v2_serving_init(context, namespace=None):
     if hasattr(context, "platform") and hasattr(
         context.platform, "set_termination_callback"
     ):
-        context.logger.debug(
+        context.logger.info(
             "Setting termination callback to terminate graph on worker shutdown"
         )
 
@@ -368,6 +368,23 @@ def v2_serving_init(context, namespace=None):
 
         context.platform.set_termination_callback(termination_callback)
 
+    if hasattr(context, "platform") and hasattr(context.platform, "set_drain_callback"):
+        context.logger.info(
+            "Setting drain callback to terminate and restart the graph on a drain event (such as rebalancing)"
+        )
+
+        def drain_callback():
+            context.logger.info("Drain callback called")
+            server.wait_for_completion()
+            context.logger.info(
+                "Termination of async flow is completed. Rerunning async flow."
+            )
+            # Rerun the flow without reconstructing it
+            server.graph._run_async_flow()
+            context.logger.info("Async flow restarted")
+
+        context.platform.set_drain_callback(drain_callback)
+
 
 def v2_serving_handler(context, event, get_body=False):
     """hook for nuclio handler()"""

mlrun/serving/states.py

@@ -921,6 +921,7 @@ class FlowStep(BaseStep):
 
         if self.engine != "sync":
             self._build_async_flow()
+            self._run_async_flow()
 
     def check_and_process_graph(self, allow_empty=False):
         """validate correct graph layout and initialize the .next links"""
@@ -1075,7 +1076,10 @@ class FlowStep(BaseStep):
                         if next_state.async_object and error_step.async_object:
                             error_step.async_object.to(next_state.async_object)
 
-        self._controller = source.run()
+        self._async_flow = source
+
+    def _run_async_flow(self):
+        self._controller = self._async_flow.run()
 
     def get_queue_links(self):
         """return dict of function and queue its listening on, for building stream triggers"""

mlrun/utils/helpers.py

@@ -65,6 +65,10 @@ DEFAULT_TIME_PARTITIONS = ["year", "month", "day", "hour"]
 DEFAULT_TIME_PARTITIONING_GRANULARITY = "hour"
 
 
+class OverwriteBuildParamsWarning(FutureWarning):
+    pass
+
+
 # TODO: remove in 1.7.0
 @deprecated(
     version="1.5.0",
@@ -183,7 +187,7 @@ def verify_field_regex(
             if mode == mlrun.common.schemas.RegexMatchModes.all:
                 if raise_on_failure:
                     raise mlrun.errors.MLRunInvalidArgumentError(
-                        f"Field '{field_name}' is malformed. {field_value} does not match required pattern: {pattern}"
+                        f"Field '{field_name}' is malformed. '{field_value}' does not match required pattern: {pattern}"
                     )
                 return False
         elif mode == mlrun.common.schemas.RegexMatchModes.any:
@@ -193,7 +197,7 @@ def verify_field_regex(
     elif mode == mlrun.common.schemas.RegexMatchModes.any:
         if raise_on_failure:
             raise mlrun.errors.MLRunInvalidArgumentError(
-                f"Field '{field_name}' is malformed. {field_value} does not match any of the"
+                f"Field '{field_name}' is malformed. '{field_value}' does not match any of the"
                 f" required patterns: {patterns}"
             )
         return False
@@ -333,7 +337,7 @@ def remove_image_protocol_prefix(image: str) -> str:
 def verify_field_of_type(field_name: str, field_value, expected_type: type):
     if not isinstance(field_value, expected_type):
         raise mlrun.errors.MLRunInvalidArgumentError(
-            f"Field '{field_name}' should be of type {expected_type.__name__} "
+            f"Field '{field_name}' should be of type '{expected_type.__name__}' "
             f"(got: {type(field_value).__name__} with value: {field_value})."
         )
 
@@ -357,14 +361,14 @@ def verify_dict_items_type(
     if dictionary:
         if type(dictionary) != dict:
             raise mlrun.errors.MLRunInvalidArgumentTypeError(
-                f"{name} expected to be of type dict, got type : {type(dictionary)}"
+                f"'{name}' expected to be of type dict, got type: {type(dictionary)}"
             )
         try:
             verify_list_items_type(dictionary.keys(), expected_keys_types)
             verify_list_items_type(dictionary.values(), expected_values_types)
         except mlrun.errors.MLRunInvalidArgumentTypeError as exc:
             raise mlrun.errors.MLRunInvalidArgumentTypeError(
-                f"{name} should be of type Dict[{get_pretty_types_names(expected_keys_types)},"
+                f"'{name}' should be of type Dict[{get_pretty_types_names(expected_keys_types)},"
                 f"{get_pretty_types_names(expected_values_types)}]."
             ) from exc
 
@@ -407,7 +411,7 @@ def normalize_name(name: str, verbose: bool = True):
         if verbose:
             warnings.warn(
                 "Names with underscore '_' are about to be deprecated, use dashes '-' instead. "
-                f"Replacing {name} underscores with dashes.",
+                f"Replacing '{name}' underscores with dashes.",
                 FutureWarning,
             )
         name = name.replace("_", "-")
@@ -669,7 +673,7 @@ def parse_artifact_uri(uri, default_project=""):
             iteration = int(iteration)
         except ValueError:
             raise ValueError(
-                f"illegal store path {uri}, iteration must be integer value"
+                f"illegal store path '{uri}', iteration must be integer value"
             )
     return (
         group_dict["project"] or default_project,
@@ -1199,7 +1203,7 @@ def get_function(function, namespace):
         function_object = create_function(function)
     except (ImportError, ValueError) as exc:
         raise ImportError(
-            f"state/function init failed, handler {function} not found"
+            f"state/function init failed, handler '{function}' not found"
         ) from exc
     return function_object
 
@@ -1383,7 +1387,7 @@ def get_in_artifact(artifact: dict, key, default=None, raise_on_missing=False):
 
         if raise_on_missing:
             raise mlrun.errors.MLRunInvalidArgumentError(
-                f"artifact {artifact} is missing metadata/spec/status"
+                f"artifact '{artifact}' is missing metadata/spec/status"
             )
         return default
 
@@ -1419,7 +1423,7 @@ def is_running_in_jupyter_notebook() -> bool:
 
 def as_number(field_name, field_value):
     if isinstance(field_value, str) and not field_value.isnumeric():
-        raise ValueError(f"{field_name} must be numeric (str/int types)")
+        raise ValueError(f"'{field_name}' must be numeric (str/int types)")
     return int(field_value)
 
 

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "4196c53355a2275531e389ac20dfb2aff559137e",
-  "version": "1.6.0-rc12"
+  "git_commit": "abca3cad961e987ec75d8ecfcdf0b6856dc9b7d1",
+  "version": "1.6.0-rc13"
 }

{mlrun-1.6.0rc12.dist-info → mlrun-1.6.0rc13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.6.0rc12
+Version: 1.6.0rc13
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -65,7 +65,7 @@ Requires-Dist: boto3 <1.29.0,>=1.28.0 ; extra == 'all'
 Requires-Dist: dask ~=2023.9.0 ; extra == 'all'
 Requires-Dist: databricks-sdk ~=0.3.0 ; extra == 'all'
 Requires-Dist: distributed ~=2023.9.0 ; extra == 'all'
-Requires-Dist: gcsfs ==2023.9.0 ; extra == 'all'
+Requires-Dist: gcsfs ==2023.9.1 ; extra == 'all'
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ~=3.2 ; extra == 'all'
 Requires-Dist: google-cloud-storage ~=1.20 ; extra == 'all'
 Requires-Dist: google-cloud ~=0.34 ; extra == 'all'
@@ -76,7 +76,7 @@ Requires-Dist: msrest ~=0.6.21 ; extra == 'all'
 Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'all'
 Requires-Dist: pyopenssl >=23 ; extra == 'all'
 Requires-Dist: redis ~=4.3 ; extra == 'all'
-Requires-Dist: s3fs ==2023.9.0 ; extra == 'all'
+Requires-Dist: s3fs ==2023.9.1 ; extra == 'all'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'all'
 Provides-Extra: api
 Requires-Dist: uvicorn ~=0.23.2 ; extra == 'api'
@@ -115,7 +115,7 @@ Requires-Dist: boto3 <1.29.0,>=1.28.0 ; extra == 'complete'
 Requires-Dist: dask ~=2023.9.0 ; extra == 'complete'
 Requires-Dist: databricks-sdk ~=0.3.0 ; extra == 'complete'
 Requires-Dist: distributed ~=2023.9.0 ; extra == 'complete'
-Requires-Dist: gcsfs ==2023.9.0 ; extra == 'complete'
+Requires-Dist: gcsfs ==2023.9.1 ; extra == 'complete'
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ~=3.2 ; extra == 'complete'
 Requires-Dist: graphviz ~=0.20.0 ; extra == 'complete'
 Requires-Dist: kafka-python ~=2.0 ; extra == 'complete'
@@ -124,7 +124,7 @@ Requires-Dist: msrest ~=0.6.21 ; extra == 'complete'
 Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'complete'
 Requires-Dist: pyopenssl >=23 ; extra == 'complete'
 Requires-Dist: redis ~=4.3 ; extra == 'complete'
-Requires-Dist: s3fs ==2023.9.0 ; extra == 'complete'
+Requires-Dist: s3fs ==2023.9.1 ; extra == 'complete'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete'
 Provides-Extra: complete-api
 Requires-Dist: adlfs ==2023.9.0 ; extra == 'complete-api'
@@ -142,7 +142,7 @@ Requires-Dist: dask ~=2023.9.0 ; extra == 'complete-api'
 Requires-Dist: databricks-sdk ~=0.3.0 ; extra == 'complete-api'
 Requires-Dist: distributed ~=2023.9.0 ; extra == 'complete-api'
 Requires-Dist: fastapi ~=0.103.2 ; extra == 'complete-api'
-Requires-Dist: gcsfs ==2023.9.0 ; extra == 'complete-api'
+Requires-Dist: gcsfs ==2023.9.1 ; extra == 'complete-api'
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ~=3.2 ; extra == 'complete-api'
 Requires-Dist: graphviz ~=0.20.0 ; extra == 'complete-api'
 Requires-Dist: humanfriendly ~=9.2 ; extra == 'complete-api'
@@ -155,7 +155,7 @@ Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'complete-api'
 Requires-Dist: pymysql ~=1.0 ; extra == 'complete-api'
 Requires-Dist: pyopenssl >=23 ; extra == 'complete-api'
 Requires-Dist: redis ~=4.3 ; extra == 'complete-api'
-Requires-Dist: s3fs ==2023.9.0 ; extra == 'complete-api'
+Requires-Dist: s3fs ==2023.9.1 ; extra == 'complete-api'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete-api'
 Requires-Dist: sqlite3-to-mysql ~=1.4 ; extra == 'complete-api'
 Requires-Dist: timelength ~=1.1 ; extra == 'complete-api'
@@ -172,7 +172,7 @@ Requires-Dist: google-cloud ~=0.34 ; extra == 'google-cloud'
 Provides-Extra: google-cloud-bigquery
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ~=3.2 ; extra == 'google-cloud-bigquery'
 Provides-Extra: google-cloud-storage
-Requires-Dist: gcsfs ==2023.9.0 ; extra == 'google-cloud-storage'
+Requires-Dist: gcsfs ==2023.9.1 ; extra == 'google-cloud-storage'
 Provides-Extra: graphviz
 Requires-Dist: graphviz ~=0.20.0 ; extra == 'graphviz'
 Provides-Extra: kafka
@@ -187,12 +187,12 @@ Requires-Dist: redis ~=4.3 ; extra == 'redis'
 Provides-Extra: s3
 Requires-Dist: boto3 <1.29.0,>=1.28.0 ; extra == 's3'
 Requires-Dist: aiobotocore <2.8,>=2.5.0 ; extra == 's3'
-Requires-Dist: s3fs ==2023.9.0 ; extra == 's3'
+Requires-Dist: s3fs ==2023.9.1 ; extra == 's3'
 Provides-Extra: sqlalchemy
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'sqlalchemy'
 
 <a id="top"></a>
-[![Build Status](https://github.com/mlrun/mlrun/workflows/CI/badge.svg)](https://github.com/mlrun/mlrun/actions)
+[![Build Status](https://github.com/mlrun/mlrun/actions/workflows/build.yaml/badge.svg?branch=development)](https://github.com/mlrun/mlrun/actions/workflows/build.yaml?query=branch%3Adevelopment)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![PyPI version fury.io](https://badge.fury.io/py/mlrun.svg)](https://pypi.python.org/pypi/mlrun/)
 [![Documentation](https://readthedocs.org/projects/mlrun/badge/?version=latest)](https://mlrun.readthedocs.io/en/latest/?badge=latest)
@@ -208,7 +208,7 @@ Requires-Dist: sqlalchemy ~=1.4 ; extra == 'sqlalchemy'
 MLRun is an open MLOps platform for quickly building and managing continuous ML applications across their lifecycle. MLRun integrates into your development and CI/CD environment and automates the delivery of production data, ML pipelines, and online applications, significantly reducing engineering efforts, time to production, and computation resources.
 With MLRun, you can choose any IDE on your local machine or on the cloud. MLRun breaks the silos between data, ML, software, and DevOps/MLOps teams, enabling collaboration and fast continuous improvements.
 
-Get started with MLRun [**Tutorials and Examples**](https://docs.mlrun.org/en/latest/tutorial/index.html), [**Installation and setup guide**](https://docs.mlrun.org/en/latest/install.html), or read about [**MLRun Architecture**](https://docs.mlrun.org/en/latest/architecture.html).
+Get started with MLRun [**Tutorials and Examples**](https://docs.mlrun.org/en/latest/tutorials/index.html), [**Installation and setup guide**](https://docs.mlrun.org/en/latest/install.html), or read about [**MLRun Architecture**](https://docs.mlrun.org/en/latest/architecture.html).
 
 This page explains how MLRun addresses the [**MLOps Tasks**](#mlops-tasks) and the [**MLRun core components**](#core-components).
 
@@ -226,32 +226,32 @@ In MLRun the assets, metadata, and services (data, functions, jobs, artifacts, m
 Projects can be imported/exported as a whole, mapped to git repositories or IDE projects (in PyCharm, VSCode, etc.), which enables versioning, collaboration, and CI/CD. 
 Project access can be restricted to a set of users and roles. 
 
-See: **Docs:** [Projects and Automation](https://docs.mlrun.org/en/latest/projects/project.html), [CI/CD Integration](https://docs.mlrun.org/en/latest/projects/ci-integration.html), **Tutorials:** [Quick start](https://docs.mlrun.org/en/latest/tutorial/01-mlrun-basics.html), [Automated ML Pipeline](https://docs.mlrun.org/en/latest/tutorial/04-pipeline.html), **Video:** [quick start](https://youtu.be/xI8KVGLlj7Q).
+See: **Docs:** [Projects and Automation](https://docs.mlrun.org/en/latest/projects/project.html), [CI/CD Integration](https://docs.mlrun.org/en/latest/projects/ci-integration.html), **Tutorials:** [Quick start](https://docs.mlrun.org/en/latest/tutorials/01-mlrun-basics.html), [Automated ML Pipeline](https://docs.mlrun.org/en/latest/tutorials/04-pipeline.html), **Video:** [quick start](https://youtu.be/xI8KVGLlj7Q).
 
 ### Ingest and process data
 
 MLRun provides abstract interfaces to various offline and online [**data sources**](https://docs.mlrun.org/en/latest/concepts/data-feature-store.html), supports batch or realtime data processing at scale, data lineage and versioning, structured and unstructured data, and more. 
 In addition, the MLRun [**Feature Store**](https://docs.mlrun.org/en/latest/feature-store/feature-store.html) automates the collection, transformation, storage, catalog, serving, and monitoring of data features across the ML lifecycle and enables feature reuse and sharing. 
 
-See: **Docs:** [Ingest and process data](https://docs.mlrun.org/en/latest/data-prep/index.html), [Feature Store](https://docs.mlrun.org/en/latest/feature-store/feature-store.html), [Data & Artifacts](https://docs.mlrun.org/en/latest/concepts/data-feature-store.html); **Tutorials:** [Quick start](https://docs.mlrun.org/en/latest/tutorial/01-mlrun-basics.html), [Feature Store](https://docs.mlrun.org/en/latest/feature-store/basic-demo.html).
+See: **Docs:** [Ingest and process data](https://docs.mlrun.org/en/latest/data-prep/index.html), [Feature Store](https://docs.mlrun.org/en/latest/feature-store/feature-store.html), [Data & Artifacts](https://docs.mlrun.org/en/latest/concepts/data-feature-store.html); **Tutorials:** [Quick start](https://docs.mlrun.org/en/latest/tutorials/01-mlrun-basics.html), [Feature Store](https://docs.mlrun.org/en/latest/feature-store/basic-demo.html).
 
 ### Develop and train models
 
 MLRun allows you to easily build ML pipelines that take data from various sources or the Feature Store and process it, train models at scale with multiple parameters, test models, tracks each experiments, register, version and deploy models, etc. MLRun provides scalable built-in or custom model training services, integrate with any framework and can work with 3rd party training/auto-ML services. You can also bring your own pre-trained model and use it in the pipeline.
 
-See: **Docs:** [Develop and train models](https://docs.mlrun.org/en/latest/development/index.html), [Model Training and Tracking](https://docs.mlrun.org/en/latest/development/model-training-tracking.html), [Batch Runs and Workflows](https://docs.mlrun.org/en/latest/concepts/runs-workflows.html); **Tutorials:** [Train & Eval Models](https://docs.mlrun.org/en/latest/tutorial/02-model-training.html), [Automated ML Pipeline](https://docs.mlrun.org/en/latest/tutorial/04-pipeline.html); **Video:** [Training models](https://youtu.be/bZgBsmLMdQo).
+See: **Docs:** [Develop and train models](https://docs.mlrun.org/en/latest/development/index.html), [Model Training and Tracking](https://docs.mlrun.org/en/latest/development/model-training-tracking.html), [Batch Runs and Workflows](https://docs.mlrun.org/en/latest/concepts/runs-workflows.html); **Tutorials:** [Train & Eval Models](https://docs.mlrun.org/en/latest/tutorials/02-model-training.html), [Automated ML Pipeline](https://docs.mlrun.org/en/latest/tutorials/04-pipeline.html); **Video:** [Training models](https://youtu.be/bZgBsmLMdQo).
 
 ### Deploy models and applications
 
 MLRun rapidly deploys and manages production-grade real-time or batch application pipelines using elastic and resilient serverless functions. MLRun addresses the entire ML application: intercepting application/user requests, running data processing tasks, inferencing using one or more models, driving actions, and integrating with the application logic.
 
-See: **Docs:** [Deploy models and applications](https://docs.mlrun.org/en/latest/deployment/index.html), [Realtime Pipelines](https://docs.mlrun.org/en/latest/serving/serving-graph.html), [Batch Inference](https://docs.mlrun.org/en/latest/concepts/TBD.html), **Tutorials:** [Realtime Serving](https://docs.mlrun.org/en/latest/tutorial/03-model-serving.html), [Batch Inference](https://docs.mlrun.org/en/latest/tutorial/07-batch-infer.html), [Advanced Pipeline](https://docs.mlrun.org/en/latest/tutorial/07-batch-infer.html); **Video:** [Serving models](https://youtu.be/OUjOus4dZfw).
+See: **Docs:** [Deploy models and applications](https://docs.mlrun.org/en/latest/deployment/index.html), [Realtime Pipelines](https://docs.mlrun.org/en/latest/serving/serving-graph.html), [Batch Inference](https://docs.mlrun.org/en/latest/concepts/TBD.html), **Tutorials:** [Realtime Serving](https://docs.mlrun.org/en/latest/tutorials/03-model-serving.html), [Batch Inference](https://docs.mlrun.org/en/latest/tutorials/07-batch-infer.html), [Advanced Pipeline](https://docs.mlrun.org/en/latest/tutorials/07-batch-infer.html); **Video:** [Serving models](https://youtu.be/OUjOus4dZfw).
 
 ### Monitor and alert
 
 Observability is built into the different MLRun objects (data, functions, jobs, models, pipelines, etc.), eliminating the need for complex integrations and code instrumentation. With MLRun, you can observe the application/model resource usage and model behavior (drift, performance, etc.), define custom app metrics, and trigger alerts or retraining jobs.
 
-See: **Docs:** [Monitor and alert](https://docs.mlrun.org/en/latest/monitoring/index.html), [Model Monitoring Overview](https://docs.mlrun.org/en/latest/monitoring/model-monitoring-deployment.html), **Tutorials:** [Model Monitoring & Drift Detection](https://docs.mlrun.org/en/latest/tutorial/05-model-monitoring.html).
+See: **Docs:** [Monitor and alert](https://docs.mlrun.org/en/latest/monitoring/index.html), [Model Monitoring Overview](https://docs.mlrun.org/en/latest/monitoring/model-monitoring-deployment.html), **Tutorials:** [Model Monitoring & Drift Detection](https://docs.mlrun.org/en/latest/tutorials/05-model-monitoring.html).
 
 
 <a id="core-components"></a>

{mlrun-1.6.0rc12.dist-info → mlrun-1.6.0rc13.dist-info}/RECORD

@@ -1,6 +1,6 @@
 mlrun/__init__.py,sha256=o9dHUfVFADfsi6GnOPLr2OkfkHdPvOnA7rkoECen0-I,7248
 mlrun/__main__.py,sha256=QdVcqPRIzc1_em8nXiO7PLkGZYvzqyQKSNoaN9Czk6Y,49066
-mlrun/config.py,sha256=-o8BrGcQolvpN_qjnRhA6r9wbGsa5A07BZHVMi_9-qI,60736
+mlrun/config.py,sha256=aXeKVEIGbsPnwIP_CVWEFCuN4S4jWr9IYRbFTvf2ZtY,60736
 mlrun/errors.py,sha256=pYZZtJsMqkFktG6T8iTyfyESrrhOLfKvB3U_jUDHMo8,6780
 mlrun/execution.py,sha256=dWus2hBaT7qj5CwsVlmNSjFYopcWW1q9ceU3AoDNW80,39708
 mlrun/features.py,sha256=UQQ2uh5Xh9XsMGiYBqh3bKgDhOHANjv1gQgWyId9qQE,15624
@@ -77,7 +77,7 @@ mlrun/datastore/s3.py,sha256=9X5_uwzPwklDHUNZ3G4LkX1law4Z-weTDCTYz8PBplk,8064
 mlrun/datastore/sources.py,sha256=3P2Q7CUP9e76BqYLv5LlVGLCp6tUsys4fs-txgh5ac8,39493
 mlrun/datastore/spark_udf.py,sha256=NnnB3DZxZb-rqpRy7b-NC7QWXuuqFn3XkBDc86tU4mQ,1498
 mlrun/datastore/store_resources.py,sha256=SUY9oJieq3r8PEq8G661XxmXem_e-CxDoy2AJ7dpXBk,6906
-mlrun/datastore/targets.py,sha256=PTcYvYM8XYYHqkuqPCPvAo87uFxpZhFof5wck5-jEWM,69463
+mlrun/datastore/targets.py,sha256=-sI5DTpPCuXTsayoyxu700iabaKFyg7u9UNubKPXxjw,69817
 mlrun/datastore/utils.py,sha256=vuuNFM1ystkvpOlvI8QJL5i8ZPKxvSO0wPv8s4W1raE,5768
 mlrun/datastore/v3io.py,sha256=ywd-rrB5Uicdk7KGMk-nJ4mKPjvg2z5w6TVx5Bo5jIo,8099
 mlrun/datastore/wasbfs/__init__.py,sha256=s5Ul-0kAhYqFjKDR2X0O2vDGDbLQQduElb32Ev56Te4,1343
@@ -95,10 +95,10 @@ mlrun/feature_store/feature_vector.py,sha256=hbEk5BzCEFxUyJPA3jefTlE95whD9c8a3f9
 mlrun/feature_store/ingestion.py,sha256=GZkrke5_JJfA_PGOFc6ekbHKujHgMgqr6t4vop5n_bg,11210
 mlrun/feature_store/steps.py,sha256=Xsvq6TGWraLeN_OC7T3ZTrJP8WhVNjDC12VSNACRePE,29118
 mlrun/feature_store/retrieval/__init__.py,sha256=bwA4copPpLQi8fyoUAYtOyrlw0-6f3-Knct8GbJSvRg,1282
-mlrun/feature_store/retrieval/base.py,sha256=06S465nMB4gOrVesnVZXN4M-01afcleX7U6sSozqoJk,31673
-mlrun/feature_store/retrieval/dask_merger.py,sha256=E_ziv7sARcnRmdNTCGV9vskRGDtPAAGsIc4PRTaL63s,5420
+mlrun/feature_store/retrieval/base.py,sha256=e7n-z3LMYMD2J13oWqgvOthnhOMpq8_MIIMuW8ZBWKg,31724
+mlrun/feature_store/retrieval/dask_merger.py,sha256=_AiEu0iRPi9nKt97EhlXqXCYfq4LgHsdpG7ZzpycReM,5491
 mlrun/feature_store/retrieval/job.py,sha256=udrpAf8Q1uNhGj3ATvoEV7M8hlZr3dW_P0T5MqKc1Dc,8262
-mlrun/feature_store/retrieval/local_merger.py,sha256=2BWCBm9lcyRfYiDRSJ42E9QjcUo7ux0URgxUHpGD-Fw,4359
+mlrun/feature_store/retrieval/local_merger.py,sha256=jkzTml_umIU-d6isKFNPtmLZ5K9upqbJLWUde2DdiL8,4429
 mlrun/feature_store/retrieval/spark_merger.py,sha256=-SaIen7Jc9RAPs4TwQ0E5qEO2Yh_Kp-YVD8b902nXAw,11079
 mlrun/feature_store/retrieval/storey_merger.py,sha256=5YM0UPrLjGOobulHkowRO-1LuvFD2cm_0GxcpnTdu0I,6314
 mlrun/frameworks/__init__.py,sha256=qRHe_nUfxpoLaSASAkIxcW6IyunMtxq5LXhjzZMO_1E,743
@@ -192,14 +192,14 @@ mlrun/launcher/factory.py,sha256=tk6foFWox7f_xaeTgkWTx9ht_5fv0XzLDR8ucdb8oTE,234
 mlrun/launcher/local.py,sha256=iSPQjnAeBmIz-oWprDmQqD9TUkz85h9aaJIbybcT1R0,10930
 mlrun/launcher/remote.py,sha256=e7orva_ozrtmvEE-QihoFi8MKNCAHxzeUNQpqwQbEoQ,7007
 mlrun/model_monitoring/__init__.py,sha256=XaYyvWsIXpjJQ2gCPj8tFvfSbRSEEqgDtNz4tCE5H4g,915
-mlrun/model_monitoring/api.py,sha256=MyS6dmFv2w9X-kGEPV4Z-Sn2Sihx85hmSe8Yd-NRQgI,34624
+mlrun/model_monitoring/api.py,sha256=7aB6EbLHeH8PMQYp8mZQfDCxZeaD391Kb-tUVhS9hSs,36674
 mlrun/model_monitoring/application.py,sha256=fgsSOLwWJWbiFV_hCOpYVTTOnZLIUDTQ0fHIeGvhMWw,12370
 mlrun/model_monitoring/batch.py,sha256=7Iq0LNbuG6yAzaZ3ut1qFMZTP2ODvGd57cufrP84wtg,43286
 mlrun/model_monitoring/controller.py,sha256=B5PNThwgjYVhjb1t1Xy2m_UIOykLF0CZHMw5mTsevew,26483
 mlrun/model_monitoring/controller_handler.py,sha256=Yk8urHYKSU_RUhO0B375VO5YWd41yvPW_SMFpO-C8vg,1095
 mlrun/model_monitoring/evidently_application.py,sha256=vlua7ikl736bFKwigCz3Qh4KV4CoWaqJ1ZVPbwCac90,3792
 mlrun/model_monitoring/features_drift_table.py,sha256=2r51W4xQ8gNq3PXt73IfsYu4l4mjwD-dLfRVAvKplTE,24209
-mlrun/model_monitoring/helpers.py,sha256=lJhH8SiRQg4jHo0HDC_89t2jghwPqOp1Xks1tFdQBj0,3269
+mlrun/model_monitoring/helpers.py,sha256=fn5bwqX-29BqecHzJLJ3JvItyZUjYVZ89p8XYbnnFhk,5589
 mlrun/model_monitoring/model_endpoint.py,sha256=BBtxdY5ciormI_al4zshmIp0GN7hGhOCn-hLgpCXek0,3938
 mlrun/model_monitoring/prometheus.py,sha256=Z0UWmhQ-dpGGH31gCiGdfmhfj-RFRf1Tu1bYVe-k4jk,7605
 mlrun/model_monitoring/stream_processing.py,sha256=LqRMosAWF1ncKjP829zDTdpkx5pPcydv5QJp3RGQY_U,48755
@@ -234,18 +234,18 @@ mlrun/platforms/__init__.py,sha256=ArWn_iZiEE6qz7hvY_1RqMkFnHGuKjP3k5xYKnfKA58,2
 mlrun/platforms/iguazio.py,sha256=LU1d33ll5EKIyp2zitCffZIbq-3fRwNSNO9MK2cIsHc,21729
 mlrun/platforms/other.py,sha256=z4pWqxXkVVuMLk-MbNb0Y_ZR5pmIsUm0R8vHnqpEnew,11852
 mlrun/projects/__init__.py,sha256=Lv5rfxyXJrw6WGOWJKhBz66M6t3_zsNMCfUD6waPwx4,1153
-mlrun/projects/operations.py,sha256=ew_wGc6GVWjX1upsh7kBMZHtWyjKglxWPFQ0Tg9o5OI,18134
+mlrun/projects/operations.py,sha256=Qn7V-ixdUDD_u21U1IwshKhAe6fTbXGruU3Ekx5-8ls,18744
 mlrun/projects/pipelines.py,sha256=qVlG5ZxdcXxdgtxeht-r6QHDUMw7DUkDPv-yuvui3Rk,39297
-mlrun/projects/project.py,sha256=2Y1dlDLY_HOqvJqBc_LRdgR9GvZrkoIBhFn3o3zxpJc,144948
+mlrun/projects/project.py,sha256=vWv-tOFWjy6OSoG0n97BHo6eAVDLXPxugQY2KMANNNY,146079
 mlrun/runtimes/__init__.py,sha256=_93Hbsamu5NDC4HTuZl6UCOQN23dJQ7CwNbkT4GAMF8,7028
 mlrun/runtimes/base.py,sha256=056I8Oh1KX-qD5hFSVmiuB2bX0ZhOztohOkomMoJ9s8,35525
 mlrun/runtimes/constants.py,sha256=Y7ZETb5-sD1m6H0dqEHmPrtvhqrwYf-uYcKgMxHFYhw,8657
 mlrun/runtimes/daskjob.py,sha256=EoTnCeQgOupl90Tg6IrQjo8Fc2I2ZwdTFMq8HD9r8Qw,19100
 mlrun/runtimes/funcdoc.py,sha256=FHwnLfFzoD6yGlsAJXAl_3VTtudgg4fTrsw_XqLOkC0,10508
-mlrun/runtimes/function.py,sha256=VI57LNEiM0mYM5xLC16HiPZRB3pUB755ZutF-q1tZyk,47204
+mlrun/runtimes/function.py,sha256=bA5-FojJXn91l8kQV6KhFzqIlwHgiaC812ne64vfsTU,47535
 mlrun/runtimes/function_reference.py,sha256=SJ0J-4ww0FQdijmdnUwGUKhMb-h5wtzqCPItTWKIL40,4911
 mlrun/runtimes/generators.py,sha256=v28HdNgxdHvj888G1dTnUeQZz-D9iTO0hoGeZbCdiuQ,7241
-mlrun/runtimes/kubejob.py,sha256=ojmFYLdECGZJbyAXMdee_U3VSkB2393AqyRIWAOrygM,12088
+mlrun/runtimes/kubejob.py,sha256=BiSnLOvcm_YvpayADtxDtqyUmuYSwdAlVo6TlHfoqus,12386
 mlrun/runtimes/local.py,sha256=wF2WZ_NyG6OKdte05Hkr085AFxJr1COI4fJfsJeHbdI,21160
 mlrun/runtimes/nuclio.py,sha256=hwk4dUaZefI-Qbb4s289vQpt1h0nAucxf6eINzVI-d8,2908
 mlrun/runtimes/pod.py,sha256=TNM8fee1lSSVHrfig0aPQRV2Vp6ICH09DhFoA6nTO-U,56312
@@ -254,8 +254,8 @@ mlrun/runtimes/serving.py,sha256=IKAv5gHrk7GXkWaVW_tvF7uPtXcdPyQnETJRVj7wXEw,303
 mlrun/runtimes/utils.py,sha256=eEbWJLC7vl2vG6bRTxsKhvbRBJuPHsf4hfM5cGHcd08,15324
 mlrun/runtimes/databricks_job/__init__.py,sha256=kXGBqhLN0rlAx0kTXhozGzFsIdSqW0uTSKMmsLgq_is,569
 mlrun/runtimes/databricks_job/databricks_cancel_task.py,sha256=lYItecKYdWkvmuE0gCLqx2OkBfC6JFp4PE3dW6WXHsI,2249
-mlrun/runtimes/databricks_job/databricks_runtime.py,sha256=3FpmWoSsRRYwLJlWJ-aTCG3UOWtX3Mqi-UvhIgLgfC8,8307
-mlrun/runtimes/databricks_job/databricks_wrapper.py,sha256=dTnUp7GJxHSaeEfVMb7zxKLfyVrGLLXgnbYA3GXgc_8,8104
+mlrun/runtimes/databricks_job/databricks_runtime.py,sha256=hbxhFqw68eRsdOu-NlxErwv55cLBE5j_m8J1Y8EfCk4,10732
+mlrun/runtimes/databricks_job/databricks_wrapper.py,sha256=MuyenBc-SdV2zz2ukCroXiRzEABClD19eCdovgcSFOc,8028
 mlrun/runtimes/mpijob/__init__.py,sha256=jZf2uPBv6IB18Jj-dGSQ9NU5_xxni7XS4dnDZGwESFE,1583
 mlrun/runtimes/mpijob/abstract.py,sha256=B9opVnv3Hdm17rFITEemYLoh9ft8q-I4wql4g12jidg,9179
 mlrun/runtimes/mpijob/v1.py,sha256=_RUlFo_3NcFf7x-QpUNVm8f7qNbRDIdUmPf_ijrv54U,3206
@@ -266,9 +266,9 @@ mlrun/serving/__init__.py,sha256=_6HRAOuS2Ehjo3vwx5h1aI_-JppxEAsl4VfEERAbGFE,107
 mlrun/serving/merger.py,sha256=PXLn3A21FiLteJHaDSLm5xKNT-80eTTjfHUJnBX1gKY,6116
 mlrun/serving/remote.py,sha256=XtCgEY-azxcP0VUG1TupZXQ_dttPkAKIAtszW-GfGpQ,18038
 mlrun/serving/routers.py,sha256=jIZIUsWaktwwzD6--02Itlgb_CCCTxi620S-jw63WhM,55310
-mlrun/serving/server.py,sha256=p0EhIF7VUnQTDtRQ-mcT-mjhov1kVfP-wDVS0tokGhA,21054
+mlrun/serving/server.py,sha256=-R_XP1NW2vG0V7F3nlPRGW0tobpL5tgZFTwcEhIZFbo,21764
 mlrun/serving/serving_wrapper.py,sha256=R670-S6PX_d5ER6jiHtRvacuPyFzQH0mEf2K0sBIIOM,836
-mlrun/serving/states.py,sha256=eh5Jq7BSKiE5QjKVNeRGI_EddnCE-DZowH_lik0_tiM,54712
+mlrun/serving/states.py,sha256=Gq7-iiFpqP1o4W1SiPdYqeyITuvnSOtz7ju46mZr3k4,54823
 mlrun/serving/utils.py,sha256=MHu3RljqtuEu3KxYcbECOzfgQZbfvYK09Tg5IAdvFgg,3902
 mlrun/serving/v1_serving.py,sha256=m_W8ylcASCvdP66lZSD6yQjamYCUJ3VZdOEQY4F8ozk,11814
 mlrun/serving/v2_serving.py,sha256=fAMA52lBvpLFpUa8LKHjRAwBZp_sR0wUmmtrN1ChJC4,21823
@@ -283,7 +283,7 @@ mlrun/utils/azure_vault.py,sha256=VNs2fz0XlFrV5Ggz3T0mR7mOWHefEcC14wM7QpsbY44,34
 mlrun/utils/clones.py,sha256=QG2ka65-ysfrOaoziudEjJqGgAxJvFKZOXkiD9WZGN4,7386
 mlrun/utils/condition_evaluator.py,sha256=oR-GjryAg76D4G79G-DzVkx631D6Gd4jJgbr_d3Btnw,1920
 mlrun/utils/db.py,sha256=2pdIYKIA0GiwwuWLW0PJ_bPu9M1rd7ESBqnMr5wWuW4,1662
-mlrun/utils/helpers.py,sha256=lyKKdQn7JqnQI9_LuNRAyvo1-iMFIVdC0O6amPoqBAY,50018
+mlrun/utils/helpers.py,sha256=lsV5lTHbqfuFXXKCBV_89kPCx1I8ao3KvHznXNEoi6M,50098
 mlrun/utils/http.py,sha256=_3pJPuDPz7M9pU4uRN-NPUmCyaANCQsAWAIrlVLZPiY,8733
 mlrun/utils/logger.py,sha256=ZVT9vnDGeuMKsbiEHTN3AXyF_hvUgpv4jGa1lgwFXJU,7049
 mlrun/utils/regex.py,sha256=V0kaw1-zuehkN20g_Pq6SgkJTBLRdBqNkXOGN_2TJEw,4430
@@ -300,11 +300,11 @@ mlrun/utils/notifications/notification/ipython.py,sha256=qrBmtECiRG6sZpCIVMg7RZc
 mlrun/utils/notifications/notification/slack.py,sha256=5JysqIpUYUZKXPSeeZtbl7qb2L9dj7p2NvnEBcEsZkA,3898
 mlrun/utils/notifications/notification/webhook.py,sha256=QHezCuN5uXkLcroAGxGrhGHaxAdUvkDLIsp27_Yrfd4,2390
 mlrun/utils/version/__init__.py,sha256=7kkrB7hEZ3cLXoWj1kPoDwo4MaswsI2JVOBpbKgPAgc,614
-mlrun/utils/version/version.json,sha256=11dCOroTQd3OhO0j2jxhnC53Rcm3CZPMrFOYj9IhoQk,89
+mlrun/utils/version/version.json,sha256=9jck-PA2eA74HRX-no59aljy7jDHijdOjUqFpYQaFiM,89
 mlrun/utils/version/version.py,sha256=HMwseV8xjTQ__6T6yUWojx_z6yUj7Io7O4NcCCH_sz8,1970
-mlrun-1.6.0rc12.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mlrun-1.6.0rc12.dist-info/METADATA,sha256=a2zOfUoFRUfX3m3QaRHkFvexPjuC7pptpbLXilwkXbk,18521
-mlrun-1.6.0rc12.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-mlrun-1.6.0rc12.dist-info/entry_points.txt,sha256=ZbXmb36B9JmK7EaleP8MIAbZSOQXQV0iwKR6si0HUWk,47
-mlrun-1.6.0rc12.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.6.0rc12.dist-info/RECORD,,
+mlrun-1.6.0rc13.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mlrun-1.6.0rc13.dist-info/METADATA,sha256=Hz1qTXPT7RuursgzNl78eUHM7ecNT6SlnCBkjkCdQ6Y,18614
+mlrun-1.6.0rc13.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+mlrun-1.6.0rc13.dist-info/entry_points.txt,sha256=ZbXmb36B9JmK7EaleP8MIAbZSOQXQV0iwKR6si0HUWk,47
+mlrun-1.6.0rc13.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.6.0rc13.dist-info/RECORD,,