mlrun 1.8.0rc31__py3-none-any.whl → 1.8.0rc33__py3-none-any.whl

mlrun/__main__.py

@@ -19,7 +19,7 @@ import socket
 import traceback
 import warnings
 from ast import literal_eval
-from base64 import b64decode, b64encode
+from base64 import b64decode
 from os import environ, path, remove
 from pprint import pprint
 
@@ -298,7 +298,7 @@ def run(
             if url_file and path.isfile(url_file):
                 with open(url_file) as fp:
                     body = fp.read()
-                based = b64encode(body.encode("utf-8")).decode("utf-8")
+                based = mlrun.utils.helpers.encode_user_code(body)
                 logger.info(f"packing code at {url_file}")
                 update_in(runtime, "spec.build.functionSourceCode", based)
                 url = f"main{pathlib.Path(url_file).suffix} {url_args}"
@@ -557,7 +557,7 @@ def build(
             exit(1)
         with open(source) as fp:
             body = fp.read()
-        based = b64encode(body.encode("utf-8")).decode("utf-8")
+        based = mlrun.utils.helpers.encode_user_code(body)
         logger.info(f"Packing code at {source}")
         b.functionSourceCode = based
         func.spec.command = ""

mlrun/config.py

@@ -30,7 +30,6 @@ import typing
 import warnings
 from collections.abc import Mapping
 from datetime import timedelta
-from distutils.util import strtobool
 from os.path import expanduser
 from threading import Lock
 
@@ -105,7 +104,7 @@ default_config = {
     # custom logger format, workes only with log_formatter: custom
     # Note that your custom format must include those 4 fields - timestamp, level, message and more
     "log_format_override": None,
-    "submit_timeout": "180",  # timeout when submitting a new k8s resource
+    "submit_timeout": "280",  # timeout when submitting a new k8s resource
     # runtimes cleanup interval in seconds
     "runtimes_cleanup_interval": "300",
     "monitoring": {
@@ -267,6 +266,7 @@ default_config = {
             # When the module is reloaded, the maximum depth recursion configuration for the recursive reload
             # function is used to prevent infinite loop
             "reload_max_recursion_depth": 100,
+            "source_code_max_bytes": 10000,
         },
         "databricks": {
             "artifact_directory_path": "/mlrun_databricks_runtime/artifacts_dictionaries"
@@ -1472,17 +1472,6 @@ def _convert_resources_to_str(config: typing.Optional[dict] = None):
             resource_requirement[resource_type] = str(value)
 
 
-def _convert_str(value, typ):
-    if typ in (str, _none_type):
-        return value
-
-    if typ is bool:
-        return strtobool(value)
-
-    # e.g. int('8080') → 8080
-    return typ(value)
-
-
 def _configure_ssl_verification(verify_ssl: bool) -> None:
     """Configure SSL verification warnings based on the setting."""
     if not verify_ssl:

mlrun/db/base.py

@@ -720,7 +720,7 @@ class RunDBInterface(ABC):
     def list_model_endpoints(
         self,
         project: str,
-        name: Optional[str] = None,
+        names: Optional[Union[str, list[str]]] = None,
         function_name: Optional[str] = None,
         function_tag: Optional[str] = None,
         model_name: Optional[str] = None,

mlrun/db/httpdb.py

@@ -3765,7 +3765,7 @@ class HTTPRunDB(RunDBInterface):
     def list_model_endpoints(
         self,
         project: str,
-        name: Optional[str] = None,
+        names: Optional[Union[str, list[str]]] = None,
         function_name: Optional[str] = None,
         function_tag: Optional[str] = None,
         model_name: Optional[str] = None,
@@ -3782,7 +3782,7 @@ class HTTPRunDB(RunDBInterface):
         List model endpoints with optional filtering by name, function name, model name, labels, and time range.
 
         :param project:         The name of the project
-        :param name:            The name of the model endpoint
+        :param names:            The name of the model endpoint, or list of names of the model endpoints
         :param function_name:   The name of the function
         :param function_tag:    The tag of the function
         :param model_name:      The name of the model
@@ -3798,12 +3798,13 @@ class HTTPRunDB(RunDBInterface):
         """
         path = f"projects/{project}/model-endpoints"
         labels = self._parse_labels(labels)
-
+        if names and isinstance(names, str):
+            names = [names]
         response = self.api_call(
             method=mlrun.common.types.HTTPMethod.GET,
             path=path,
             params={
-                "name": name,
+                "name": names,
                 "model_name": model_name,
                 "model_tag": model_tag,
                 "function_name": function_name,

mlrun/db/nopdb.py

@@ -617,7 +617,7 @@ class NopDB(RunDBInterface):
     def list_model_endpoints(
         self,
         project: str,
-        name: Optional[str] = None,
+        names: Optional[Union[str, list[str]]] = None,
         function_name: Optional[str] = None,
         function_tag: Optional[str] = None,
         model_name: Optional[str] = None,

mlrun/model_monitoring/api.py

@@ -88,19 +88,24 @@ def get_or_create_model_endpoint(
         # Generate a runtime database
         db_session = mlrun.get_run_db()
     model_endpoint = None
+    if not function_name and context:
+        function_name = FunctionURI.from_string(
+            context.to_dict()["spec"]["function"]
+        ).function
     try:
-        if endpoint_id:
-            model_endpoint = db_session.get_model_endpoint(
-                project=project,
-                name=model_endpoint_name,
-                endpoint_id=endpoint_id,
-            )
-            # If other fields provided, validate that they are correspond to the existing model endpoint data
-            _model_endpoint_validations(
-                model_endpoint=model_endpoint,
-                model_path=model_path,
-                sample_set_statistics=sample_set_statistics,
-            )
+        model_endpoint = db_session.get_model_endpoint(
+            project=project,
+            name=model_endpoint_name,
+            endpoint_id=endpoint_id,
+            function_name=function_name,
+            function_tag=function_tag or "latest",
+        )
+        # If other fields provided, validate that they are correspond to the existing model endpoint data
+        _model_endpoint_validations(
+            model_endpoint=model_endpoint,
+            model_path=model_path,
+            sample_set_statistics=sample_set_statistics,
+        )
 
     except mlrun.errors.MLRunNotFoundError:
         # Create a new model endpoint with the provided details
@@ -361,10 +366,6 @@ def _generate_model_endpoint(
 
     :return `mlrun.common.schemas.ModelEndpoint` object.
     """
-    if not function_name and context:
-        function_name = FunctionURI.from_string(
-            context.to_dict()["spec"]["function"]
-        ).function
     model_obj = None
     if model_path:
         model_obj: mlrun.artifacts.ModelArtifact = (

mlrun/model_monitoring/applications/__init__.py

@@ -15,9 +15,4 @@
 
 from .base import ModelMonitoringApplicationBase
 from .context import MonitoringApplicationContext
-from .evidently_base import (
-    _HAS_EVIDENTLY,
-    SUPPORTED_EVIDENTLY_VERSION,
-    EvidentlyModelMonitoringApplicationBase,
-)
 from .results import ModelMonitoringApplicationMetric, ModelMonitoringApplicationResult

mlrun/model_monitoring/applications/base.py

@@ -28,6 +28,7 @@ import mlrun.model_monitoring.api as mm_api
 import mlrun.model_monitoring.applications.context as mm_context
 import mlrun.model_monitoring.applications.results as mm_results
 from mlrun.serving.utils import MonitoringApplicationToDict
+from mlrun.utils import logger
 
 
 class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
@@ -94,7 +95,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         context: "mlrun.MLClientCtx",
         sample_data: Optional[pd.DataFrame] = None,
         reference_data: Optional[pd.DataFrame] = None,
-        endpoints: Optional[list[tuple[str, str]]] = None,
+        endpoints: Optional[Union[list[tuple[str, str]], list[str], str]] = None,
         start: Optional[str] = None,
         end: Optional[str] = None,
         base_period: Optional[int] = None,
@@ -137,14 +138,62 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                         }
                     )
                     result_key = (
-                        f"{endpoint_name}_{window_start.isoformat()}_{window_end.isoformat()}"
+                        f"{endpoint_name}-{endpoint_id}_{window_start.isoformat()}_{window_end.isoformat()}"
                         if window_start and window_end
-                        else endpoint_name
+                        else f"{endpoint_name}-{endpoint_id}"
                     )
                     context.log_result(result_key, result)
         else:
             return call_do_tracking()
 
+    @staticmethod
+    def _handle_endpoints_type_evaluate(
+        project: str,
+        endpoints: Union[list[tuple[str, str]], list[str], str, None],
+    ) -> list[tuple[str, str]]:
+        if endpoints:
+            if isinstance(endpoints, str) or (
+                isinstance(endpoints, list) and isinstance(endpoints[0], str)
+            ):
+                endpoints_list = (
+                    mlrun.get_run_db()
+                    .list_model_endpoints(
+                        project,
+                        names=endpoints,
+                        latest_only=True,
+                    )
+                    .endpoints
+                )
+                if endpoints_list:
+                    list_endpoints_result = [
+                        (endpoint.metadata.name, endpoint.metadata.uid)
+                        for endpoint in endpoints_list
+                    ]
+                    retrieve_ep_names = list(
+                        map(lambda endpoint: endpoint[0], list_endpoints_result)
+                    )
+                    missing = set(
+                        [endpoints] if isinstance(endpoints, str) else endpoints
+                    ) - set(retrieve_ep_names)
+                    if missing:
+                        logger.warning(
+                            "Could not list all the required endpoints.",
+                            missing_endpoint=missing,
+                            endpoints=list_endpoints_result,
+                        )
+                else:
+                    raise mlrun.errors.MLRunNotFoundError(
+                        f"Did not find any model_endpoint named ' {endpoints}'"
+                    )
+
+            if not (
+                isinstance(endpoints, list) and isinstance(endpoints[0], (list, tuple))
+            ):
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "Could not resolve endpoints as list of [(name, uid)]"
+                )
+        return endpoints
+
     @staticmethod
     def _window_generator(
         start: Optional[str], end: Optional[str], base_period: Optional[int]
@@ -337,7 +386,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         class_handler: Optional[str] = None,
         requirements: Optional[Union[str, list[str]]] = None,
         requirements_file: str = "",
-        endpoints: Optional[list[tuple[str, str]]] = None,
+        endpoints: Optional[Union[list[tuple[str, str]], list[str], str]] = None,
         start: Optional[datetime] = None,
         end: Optional[datetime] = None,
         base_period: Optional[int] = None,
@@ -365,24 +414,29 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         :param requirements:      List of Python requirements to be installed in the image.
         :param requirements_file: Path to a Python requirements file to be installed in the image.
         :param endpoints:         A list of tuples of the model endpoint (name, uid) to get the data from.
-                                  If provided, and ``sample_data`` is not, you have to provide also the ``start`` and
-                                  ``end`` times of the data to analyze from the model endpoints.
+                                  allow providing a list of model_endpoint names or name for a single model_endpoint.
+                                  Note: provide names retrieves the model all the active model endpoints using those
+                                  names (cross function model endpoints)
+                                  If provided, and ``sample_data`` is not ``None``, you have to provide also the
+                                  ``start`` and ``end`` times of the data to analyze from the model endpoints.
         :param start:             The start time of the endpoint's data, not included.
                                   If you want the model endpoint's data at ``start`` included, you need to subtract a
                                   small ``datetime.timedelta`` from it.
         :param end:               The end time of the endpoint's data, included.
                                   Please note: when ``start`` and ``end`` are set, they create a left-open time interval
-                                  ("window") :math:`(\\text{start}, \\text{end}]` that excludes the endpoint's data at
-                                  ``start`` and includes the data at ``end``:
-                                  :math:`\\text{start} < t \\leq \\text{end}`, :math:`t` is the time taken in the
-                                  window's data.
+                                  ("window") :math:`(\\operatorname{start}, \\operatorname{end}]` that excludes the
+                                  endpoint's data at ``start`` and includes the data at ``end``:
+                                  :math:`\\operatorname{start} < t \\leq \\operatorname{end}`, :math:`t` is the time
+                                  taken in the window's data.
         :param base_period:       The window length in minutes. If ``None``, the whole window from ``start`` to ``end``
                                   is taken. If an integer is specified, the application is run from ``start`` to ``end``
                                   in ``base_period`` length windows, except for the last window that ends at ``end`` and
                                   therefore may be shorter:
-                                  :math:`(\\text{start}, \\text{start} + \\text{base_period}],
-                                  (\\text{start} + \\text{base_period}, \\text{start} + 2\\cdot\\text{base_period}],
-                                  ..., (\\text{start} + m\\cdot\\text{base_period}, \\text{end}]`,
+                                  :math:`(\\operatorname{start}, \\operatorname{start} + \\operatorname{base\\_period}],
+                                  (\\operatorname{start} + \\operatorname{base\\_period},
+                                  \\operatorname{start} + 2\\cdot\\operatorname{base\\_period}],
+                                  ..., (\\operatorname{start} +
+                                  m\\cdot\\operatorname{base\\_period}, \\operatorname{end}]`,
                                   where :math:`m` is some positive integer.
 
         :returns: The output of the
@@ -405,6 +459,10 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
 
         params: dict[str, Union[list[tuple[str, str]], str, int, None]] = {}
         if endpoints:
+            endpoints = cls._handle_endpoints_type_evaluate(
+                project=project.name,
+                endpoints=endpoints,
+            )
             params["endpoints"] = endpoints
             if sample_data is None:
                 if start is None or end is None:

mlrun/model_monitoring/applications/evidently/__init__.py

@@ -0,0 +1,19 @@
+# Copyright 2025 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .base import (
+    _HAS_EVIDENTLY,
+    SUPPORTED_EVIDENTLY_VERSION,
+    EvidentlyModelMonitoringApplicationBase,
+)

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 import asyncio
 from datetime import datetime, timedelta
+from threading import Lock
 from typing import Callable, Literal, Optional, Union
 
 import pandas as pd
@@ -30,6 +31,9 @@ from mlrun.model_monitoring.db import TSDBConnector
 from mlrun.model_monitoring.helpers import get_invocations_fqn
 from mlrun.utils import logger
 
+_connection = None
+_connection_lock = Lock()
+
 
 class TDEngineConnector(TSDBConnector):
     """
@@ -37,23 +41,18 @@ class TDEngineConnector(TSDBConnector):
     """
 
     type: str = mm_schemas.TSDBTarget.TDEngine
+    database = f"{tdengine_schemas._MODEL_MONITORING_DATABASE}_{mlrun.mlconf.system_id}"
 
     def __init__(
         self,
         project: str,
         profile: DatastoreProfile,
-        database: Optional[str] = None,
         **kwargs,
     ):
         super().__init__(project=project)
 
         self._tdengine_connection_profile = profile
-        self.database = (
-            database
-            or f"{tdengine_schemas._MODEL_MONITORING_DATABASE}_{mlrun.mlconf.system_id}"
-        )
 
-        self._connection = None
         self._init_super_tables()
 
         self._timeout = mlrun.mlconf.model_endpoint_monitoring.tdengine.timeout
@@ -61,9 +60,16 @@ class TDEngineConnector(TSDBConnector):
 
     @property
     def connection(self) -> TDEngineConnection:
-        if not self._connection:
-            self._connection = self._create_connection()
-        return self._connection
+        global _connection
+
+        if _connection:
+            return _connection
+
+        with _connection_lock:
+            if not _connection:
+                _connection = self._create_connection()
+
+        return _connection
 
     def _create_connection(self) -> TDEngineConnection:
         """Establish a connection to the TSDB server."""
@@ -99,7 +105,8 @@ class TDEngineConnector(TSDBConnector):
         """Create TDEngine supertables."""
         for table in self.tables:
             create_table_query = self.tables[table]._create_super_table_query()
-            self.connection.run(
+            conn = self.connection
+            conn.run(
                 statements=create_table_query,
                 timeout=self._timeout,
                 retries=self._retries,

mlrun/model_monitoring/helpers.py

@@ -141,7 +141,8 @@ def get_stream_path(
     elif isinstance(
         profile, mlrun.datastore.datastore_profile.DatastoreProfileKafkaSource
     ):
-        stream_uri = f"kafka://{profile.brokers[0]}"
+        attributes = profile.attributes()
+        stream_uri = f"kafka://{attributes['brokers'][0]}"
     else:
         raise mlrun.errors.MLRunValueError(
             f"Received an unexpected stream profile type: {type(profile)}\n"

mlrun/projects/pipelines.py

@@ -629,7 +629,7 @@ class _KFPRunner(_PipelineRunner):
                     secret_params=notification.secret_params,
                 )
 
-            project.spec.notifications = project.notifiers.server_notifications
+        project.spec.notifications = project.notifiers.server_notifications
 
         run_id = _run_pipeline(
             workflow_handler,

mlrun/projects/project.py

@@ -2155,7 +2155,7 @@ class MlrunProject(ModelObj):
 
                                        For example:
                                        [`app1.result-*`, `*.result1`]
-                                       will match "mep1.app1.result.result-1" and "mep1.app2.result.result1".
+                                       will match "mep_uid1.app1.result.result-1" and "mep_uid1.app2.result.result1".
                                        A specific result_name (not a wildcard) will always create a new alert
                                        config, regardless of whether the result name exists.
         :param severity:               Severity of the alert.
@@ -3786,7 +3786,7 @@ class MlrunProject(ModelObj):
 
     def list_model_endpoints(
         self,
-        name: Optional[str] = None,
+        names: Optional[Union[str, list[str]]] = None,
         model_name: Optional[str] = None,
         model_tag: Optional[str] = None,
         function_name: Optional[str] = None,
@@ -3816,7 +3816,7 @@ class MlrunProject(ModelObj):
         In addition, this functions provides a facade for listing endpoint related metrics. This facade is time-based
         and depends on the 'start' and 'end' parameters.
 
-        :param name: The name of the model to filter by
+        :param names: The name of the model to filter by
         :param model_name: The name of the model to filter by
         :param function_name: The name of the function to filter by
         :param function_tag: The tag of the function to filter by
@@ -3837,7 +3837,7 @@ class MlrunProject(ModelObj):
         db = mlrun.db.get_run_db(secrets=self._secrets)
         return db.list_model_endpoints(
             project=self.name,
-            name=name,
+            names=names,
             model_name=model_name,
             model_tag=model_tag,
             function_name=function_name,

mlrun/runtimes/base.py

@@ -16,7 +16,6 @@ import http
 import re
 import typing
 import warnings
-from base64 import b64encode
 from os import environ
 from typing import Callable, Optional, Union
 
@@ -795,9 +794,7 @@ class BaseRuntime(ModelObj):
                     mlrun.runtimes.nuclio.serving.serving_subkind
                 )
 
-        self.spec.build.functionSourceCode = b64encode(body.encode("utf-8")).decode(
-            "utf-8"
-        )
+        self.spec.build.functionSourceCode = mlrun.utils.helpers.encode_user_code(body)
         if with_doc:
             update_function_entry_points(self, body)
         return self

mlrun/runtimes/databricks_job/databricks_runtime.py

@@ -13,12 +13,13 @@
 # limitations under the License.
 
 from ast import FunctionDef, parse, unparse
-from base64 import b64decode, b64encode
+from base64 import b64decode
 from typing import Callable, Optional, Union
 
 import mlrun
 import mlrun.runtimes.kubejob as kubejob
 import mlrun.runtimes.pod as pod
+import mlrun.utils.helpers
 from mlrun.errors import MLRunInvalidArgumentError
 from mlrun.model import HyperParamOptions, RunObject
 
@@ -162,7 +163,7 @@ class DatabricksRuntime(kubejob.KubejobRuntime):
         if original_handler:
             decoded_code += f"\nresult = {original_handler}(**handler_arguments)\n"
             decoded_code += _return_artifacts_code
-        return b64encode(decoded_code.encode("utf-8")).decode("utf-8")
+        return mlrun.utils.helpers.encode_user_code(decoded_code)
 
     def get_internal_parameters(self, runobj: RunObject):
         """
@@ -202,7 +203,7 @@ from mlrun.runtimes.databricks_job import databricks_wrapper
 def run_mlrun_databricks_job(context,task_parameters: dict, **kwargs):
         databricks_wrapper.run_mlrun_databricks_job(context, task_parameters, **kwargs)
 """
-        wrap_code = b64encode(wrap_code).decode("utf-8")
+        wrap_code = mlrun.utils.helpers.encode_user_code(wrap_code)
         self.spec.build.functionSourceCode = wrap_code
         runspec.spec.handler = "run_mlrun_databricks_job"
 

mlrun/runtimes/function_reference.py

@@ -13,11 +13,11 @@
 # limitations under the License.
 
 import os
-from base64 import b64encode
 
 from nuclio.build import mlrun_footer
 
 import mlrun
+import mlrun.utils.helpers
 
 from ..model import ModelObj
 from ..utils import generate_object_uri
@@ -116,7 +116,7 @@ class FunctionReference(ModelObj):
             func = mlrun.new_function(
                 self.name, kind=kind, image=self.image or default_image
             )
-            data = b64encode(code.encode("utf-8")).decode("utf-8")
+            data = mlrun.utils.helpers.encode_user_code(code)
             func.spec.build.functionSourceCode = data
             if kind not in mlrun.runtimes.RuntimeKinds.nuclio_runtimes():
                 func.spec.default_handler = "handler"

mlrun/runtimes/pod.py

@@ -17,6 +17,7 @@ import os
 import re
 import time
 import typing
+import warnings
 from collections.abc import Iterable
 from enum import Enum
 
@@ -703,29 +704,7 @@ class KubeResourceSpec(FunctionSpec):
                 ),
                 affinity_field_name=affinity_field_name,
             )
-        # purge any affinity / anti-affinity preemption related configuration and enrich with preemptible tolerations
         elif self_preemption_mode == PreemptionModes.allow.value:
-            # remove preemptible anti-affinity
-            self._prune_affinity_node_selector_requirement(
-                generate_preemptible_node_selector_requirements(
-                    NodeSelectorOperator.node_selector_op_not_in.value
-                ),
-                affinity_field_name=affinity_field_name,
-            )
-            # remove preemptible affinity
-            self._prune_affinity_node_selector_requirement(
-                generate_preemptible_node_selector_requirements(
-                    NodeSelectorOperator.node_selector_op_in.value
-                ),
-                affinity_field_name=affinity_field_name,
-            )
-
-            # remove preemptible nodes constrain
-            self._prune_node_selector(
-                mlconf.get_preemptible_node_selector(),
-                node_selector_field_name=node_selector_field_name,
-            )
-
             # enrich with tolerations
             self._merge_tolerations(
                 generate_preemptible_tolerations(),
@@ -1201,6 +1180,132 @@ class KubeResource(BaseRuntime):
         """
         self.spec.with_requests(mem, cpu, patch=patch)
 
+    def detect_preemptible_node_selector(
+        self, node_selector: dict[str, str]
+    ) -> list[str]:
+        """
+        Checks if any provided node selector matches the preemptible node selectors.
+        Issues a warning if a selector may be pruned at runtime depending on preemption mode.
+
+        :param node_selector: The user-provided node selector dictionary.
+        """
+        preemptible_node_selector = mlconf.get_preemptible_node_selector()
+
+        return [
+            f"'{key}': '{val}'"
+            for key, val in node_selector.items()
+            if preemptible_node_selector.get(key) == val
+        ]
+
+    def detect_preemptible_tolerations(
+        self, tolerations: list[k8s_client.V1Toleration]
+    ) -> list[str]:
+        """
+        Checks if any provided toleration matches preemptible tolerations.
+        Issues a warning if a toleration may be pruned at runtime depending on preemption mode.
+
+        :param tolerations: The user-provided list of tolerations.
+        """
+        preemptible_tolerations = [
+            k8s_client.V1Toleration(
+                key=toleration.get("key"),
+                value=toleration.get("value"),
+                effect=toleration.get("effect"),
+            )
+            for toleration in mlconf.get_preemptible_tolerations()
+        ]
+
+        def _format_toleration(toleration):
+            return f"'{toleration.key}'='{toleration.value}' (effect: '{toleration.effect}')"
+
+        return [
+            _format_toleration(toleration)
+            for toleration in tolerations
+            if toleration in preemptible_tolerations
+        ]
+
+    def detect_preemptible_affinity(self, affinity: k8s_client.V1Affinity) -> list[str]:
+        """
+        Checks if any provided affinity rules match preemptible affinity configurations.
+        Issues a warning if an affinity rule may be pruned at runtime depending on preemption mode.
+
+        :param affinity: The user-provided affinity object.
+        """
+
+        preemptible_affinity_terms = generate_preemptible_nodes_affinity_terms()
+        conflicting_affinities = []
+
+        if (
+            affinity
+            and affinity.node_affinity
+            and affinity.node_affinity.required_during_scheduling_ignored_during_execution
+        ):
+            user_terms = affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms
+            for user_term in user_terms:
+                user_expressions = {
+                    (expr.key, expr.operator, tuple(expr.values or []))
+                    for expr in user_term.match_expressions or []
+                }
+
+                for preemptible_term in preemptible_affinity_terms:
+                    preemptible_expressions = {
+                        (expr.key, expr.operator, tuple(expr.values or []))
+                        for expr in preemptible_term.match_expressions or []
+                    }
+
+                    # Ensure operators match and preemptible expressions are present
+                    common_exprs = user_expressions & preemptible_expressions
+                    if common_exprs:
+                        formatted = ", ".join(
+                            f"'{key}  {operator}  {list(values)}'"
+                            for key, operator, values in common_exprs
+                        )
+                        conflicting_affinities.append(formatted)
+        return conflicting_affinities
+
+    def raise_preemptible_warning(
+        self,
+        node_selector: typing.Optional[dict[str, str]],
+        tolerations: typing.Optional[list[k8s_client.V1Toleration]],
+        affinity: typing.Optional[k8s_client.V1Affinity],
+    ) -> None:
+        """
+        Detects conflicts and issues a single warning if necessary.
+
+        :param node_selector: The user-provided node selector dictionary.
+        :param tolerations: The user-provided list of tolerations.
+        :param affinity: The user-provided affinity object.
+        """
+        conflict_messages = []
+
+        if node_selector:
+            ns_conflicts = ", ".join(
+                self.detect_preemptible_node_selector(node_selector)
+            )
+            if ns_conflicts:
+                conflict_messages.append(f"Node selectors: {ns_conflicts}")
+
+        if tolerations:
+            tol_conflicts = ", ".join(self.detect_preemptible_tolerations(tolerations))
+            if tol_conflicts:
+                conflict_messages.append(f"Tolerations: {tol_conflicts}")
+
+        if affinity:
+            affinity_conflicts = ", ".join(self.detect_preemptible_affinity(affinity))
+            if affinity_conflicts:
+                conflict_messages.append(f"Affinity: {affinity_conflicts}")
+
+        if conflict_messages:
+            warning_componentes = "; \n".join(conflict_messages)
+            warnings.warn(
+                f"Warning: based on the preemptible node settings configured in your MLRun configuration,\n"
+                f"{warning_componentes}\n"
+                f" may be removed or adjusted at runtime.\n"
+                "This adjustment depends on the function's preemption mode. \n"
+                "The list of potential adjusted preemptible selectors can be viewed here: "
+                "mlrun.mlconf.get_preemptible_node_selector() and mlrun.mlconf.get_preemptible_tolerations()."
+            )
+
     def with_node_selection(
         self,
         node_name: typing.Optional[str] = None,
@@ -1209,19 +1314,14 @@ class KubeResource(BaseRuntime):
         tolerations: typing.Optional[list[k8s_client.V1Toleration]] = None,
     ):
         """
-        Enables to control on which k8s node the job will run
-
-        :param node_name:       The name of the k8s node
-        :param node_selector:   Label selector, only nodes with matching labels will be eligible to be picked
-        :param affinity:        Expands the types of constraints you can express - see
-                                https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity
-                                for details
-        :param tolerations:     Tolerations are applied to pods, and allow (but do not require) the pods to schedule
-                                onto nodes with matching taints - see
-                                https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration
-                                for details
+        Enables control over which Kubernetes node the job will run on.
 
+        :param node_name:       The name of the Kubernetes node.
+        :param node_selector:   Label selector, only nodes with matching labels will be eligible.
+        :param affinity:        Defines scheduling constraints.
+        :param tolerations:     Allows scheduling onto nodes with matching taints.
         """
+        # Apply values as before
         if node_name:
             self.spec.node_name = node_name
         if node_selector is not None:
@@ -1232,6 +1332,12 @@ class KubeResource(BaseRuntime):
         if tolerations is not None:
             self.spec.tolerations = tolerations
 
+        self.raise_preemptible_warning(
+            node_selector=self.spec.node_selector,
+            tolerations=self.spec.tolerations,
+            affinity=self.spec.affinity,
+        )
+
     def with_priority_class(self, name: typing.Optional[str] = None):
         """
         Enables to control the priority of the pod

mlrun/utils/helpers.py

@@ -1329,7 +1329,11 @@ def get_handler_extended(
 def datetime_from_iso(time_str: str) -> Optional[datetime]:
     if not time_str:
         return
-    return parser.isoparse(time_str)
+    dt = parser.isoparse(time_str)
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    # ensure the datetime is in UTC, converting if necessary
+    return dt.astimezone(timezone.utc)
 
 
 def datetime_to_iso(time_obj: Optional[datetime]) -> Optional[str]:
@@ -1459,6 +1463,16 @@ def str_to_timestamp(time_str: str, now_time: Timestamp = None):
     return Timestamp(time_str)
 
 
+def str_to_bool(value: str) -> bool:
+    """Convert a string to a boolean value."""
+    value = value.lower()
+    if value in ("true", "1", "t", "y", "yes", "on"):
+        return True
+    if value in ("false", "0", "f", "n", "no", "off"):
+        return False
+    raise ValueError(f"invalid boolean value: {value}")
+
+
 def is_link_artifact(artifact):
     if isinstance(artifact, dict):
         return (
@@ -2129,3 +2143,16 @@ def as_dict(data: typing.Union[dict, str]) -> dict:
     if isinstance(data, str):
         return json.loads(data)
     return data
+
+
+def encode_user_code(
+    user_code: str, max_len_warning: typing.Optional[int] = None
+) -> str:
+    max_len_warning = max_len_warning or config.function.spec.source_code_max_bytes
+    encoded = base64.b64encode(user_code.encode("utf-8")).decode("utf-8")
+    if len(encoded) > max_len_warning:
+        logger.warning(
+            f"User code exceeds the maximum allowed size of {max_len_warning} bytes for non remote source. "
+            "Consider using `with_source_archive` to add user code as a remote source to the function."
+        )
+    return encoded

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "d59051b8a82bcd6d655bf0194284070a4a7c6341",
-  "version": "1.8.0-rc31"
+  "git_commit": "35557735bf140b7a1333fb726dd47e5b48002bc0",
+  "version": "1.8.0-rc33"
 }

{mlrun-1.8.0rc31.dist-info → mlrun-1.8.0rc33.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mlrun
-Version: 1.8.0rc31
+Version: 1.8.0rc33
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -51,8 +51,8 @@ Requires-Dist: setuptools>=75.2
 Requires-Dist: deprecated~=1.2
 Requires-Dist: jinja2>=3.1.3,~=3.1
 Requires-Dist: orjson<4,>=3.9.15
-Requires-Dist: mlrun-pipelines-kfp-common~=0.3.9
-Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.3.5; python_version < "3.11"
+Requires-Dist: mlrun-pipelines-kfp-common~=0.3.11
+Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.3.7; python_version < "3.11"
 Requires-Dist: docstring_parser~=0.16
 Requires-Dist: aiosmtplib~=3.0
 Provides-Extra: s3
@@ -99,7 +99,7 @@ Requires-Dist: ossfs==2023.12.0; extra == "alibaba-oss"
 Requires-Dist: oss2==2.18.1; extra == "alibaba-oss"
 Provides-Extra: tdengine
 Requires-Dist: taos-ws-py==0.3.2; extra == "tdengine"
-Requires-Dist: taoswswrap~=0.3.0; extra == "tdengine"
+Requires-Dist: taoswswrap~=0.3.2; extra == "tdengine"
 Provides-Extra: snowflake
 Requires-Dist: snowflake-connector-python~=3.7; extra == "snowflake"
 Provides-Extra: kfp18
@@ -119,7 +119,7 @@ Requires-Dist: timelength~=1.1; extra == "api"
 Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "api"
 Requires-Dist: aiosmtplib~=3.0; extra == "api"
 Requires-Dist: pydantic<2,>=1; extra == "api"
-Requires-Dist: mlrun-pipelines-kfp-v1-8[kfp]~=0.3.5; python_version < "3.11" and extra == "api"
+Requires-Dist: mlrun-pipelines-kfp-v1-8[kfp]~=0.3.7; python_version < "3.11" and extra == "api"
 Requires-Dist: grpcio~=1.70.0; extra == "api"
 Provides-Extra: all
 Requires-Dist: adlfs==2023.9.0; extra == "all"
@@ -152,7 +152,7 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "all"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "all"
 Requires-Dist: sqlalchemy~=1.4; extra == "all"
 Requires-Dist: taos-ws-py==0.3.2; extra == "all"
-Requires-Dist: taoswswrap~=0.3.0; extra == "all"
+Requires-Dist: taoswswrap~=0.3.2; extra == "all"
 Provides-Extra: complete
 Requires-Dist: adlfs==2023.9.0; extra == "complete"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete"
@@ -184,7 +184,7 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "complete"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "complete"
 Requires-Dist: sqlalchemy~=1.4; extra == "complete"
 Requires-Dist: taos-ws-py==0.3.2; extra == "complete"
-Requires-Dist: taoswswrap~=0.3.0; extra == "complete"
+Requires-Dist: taoswswrap~=0.3.2; extra == "complete"
 Provides-Extra: complete-api
 Requires-Dist: adlfs==2023.9.0; extra == "complete-api"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete-api"
@@ -215,7 +215,7 @@ Requires-Dist: igz-mgmt~=0.4.1; extra == "complete-api"
 Requires-Dist: kafka-python~=2.0; extra == "complete-api"
 Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "complete-api"
 Requires-Dist: mlflow~=2.16; extra == "complete-api"
-Requires-Dist: mlrun-pipelines-kfp-v1-8[kfp]~=0.3.5; python_version < "3.11" and extra == "complete-api"
+Requires-Dist: mlrun-pipelines-kfp-v1-8[kfp]~=0.3.7; python_version < "3.11" and extra == "complete-api"
 Requires-Dist: msrest~=0.6.21; extra == "complete-api"
 Requires-Dist: objgraph~=3.6; extra == "complete-api"
 Requires-Dist: oss2==2.18.1; extra == "complete-api"
@@ -229,7 +229,7 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "complete-api"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "complete-api"
 Requires-Dist: sqlalchemy~=1.4; extra == "complete-api"
 Requires-Dist: taos-ws-py==0.3.2; extra == "complete-api"
-Requires-Dist: taoswswrap~=0.3.0; extra == "complete-api"
+Requires-Dist: taoswswrap~=0.3.2; extra == "complete-api"
 Requires-Dist: timelength~=1.1; extra == "complete-api"
 Requires-Dist: uvicorn~=0.32.1; extra == "complete-api"
 Dynamic: author

{mlrun-1.8.0rc31.dist-info → mlrun-1.8.0rc33.dist-info}/RECORD

@@ -1,6 +1,6 @@
 mlrun/__init__.py,sha256=Cqm9U9eCEdLpMejhU2BEhubu0mHL71igJJIwYa738EA,7450
-mlrun/__main__.py,sha256=ysteSDo1LYe_YOXVdIVEJ3BhLPOfBngkEfRg5iaGGg4,46202
-mlrun/config.py,sha256=bW8K3MHK7jommbncuDr5N5EM9yoBqT4kA4DjKQ_1vxA,71264
+mlrun/__main__.py,sha256=xYWflUbfSRpo8F4uzrHhBN1dcaBUADmfoK5Q-iqEBeQ,46181
+mlrun/config.py,sha256=hATfrO5ZtykMiDJ6TNtkIDVgq6YwcgVrZf7H3doh5UE,71077
 mlrun/errors.py,sha256=LkcbXTLANGdsgo2CRX2pdbyNmt--lMsjGv0XZMgP-Nc,8222
 mlrun/execution.py,sha256=FUktsD3puSFjc3LZJU35b-OmFBrBPBNntViCLQVuwnk,50008
 mlrun/features.py,sha256=ReBaNGsBYXqcbgI012n-SO_j6oHIbk_Vpv0CGPXbUmo,15842
@@ -108,10 +108,10 @@ mlrun/datastore/wasbfs/__init__.py,sha256=s5Ul-0kAhYqFjKDR2X0O2vDGDbLQQduElb32Ev
 mlrun/datastore/wasbfs/fs.py,sha256=ge8NK__5vTcFT-krI155_8RDUywQw4SIRX6BWATXy9Q,6299
 mlrun/db/__init__.py,sha256=WqJ4x8lqJ7ZoKbhEyFqkYADd9P6E3citckx9e9ZLcIU,1163
 mlrun/db/auth_utils.py,sha256=hpg8D2r82oN0BWabuWN04BTNZ7jYMAF242YSUpK7LFM,5211
-mlrun/db/base.py,sha256=jVM_M5_K0EyOSEwxQokGTM6dKKsMcy5LnFGDtVpKkgc,30678
+mlrun/db/base.py,sha256=pFf33ey6vlTN7cEr8EpsSQhdydDhSXtDz5ukK5o25f8,30697
 mlrun/db/factory.py,sha256=yP2vVmveUE7LYTCHbS6lQIxP9rW--zdISWuPd_I3d_4,2111
-mlrun/db/httpdb.py,sha256=I_UiwFtwF5kUvyS1aJyF67vCCQ1oPey4J6yo2ApRtW0,230195
-mlrun/db/nopdb.py,sha256=dtyUqnleBN7i59xfgjMB4e-aGM_Oj6ONOIzMKxG3UlM,27113
+mlrun/db/httpdb.py,sha256=Pl9BdXDUJRLfwLdOYgbh-Bs-wxUoE-9v5l2gXWH7Mec,230329
+mlrun/db/nopdb.py,sha256=h_tgZDWr-7ri_gbgD3FOb9i4X_W9e4rr7va7S10kInI,27132
 mlrun/feature_store/__init__.py,sha256=AVnY2AFUNc2dKxLLUMx2K3Wo1eGviv0brDcYlDnmtf4,1506
 mlrun/feature_store/api.py,sha256=qkojZpzqGAn3r9ww0ynBRKOs8ji8URaK4DSYD4SE-CE,50395
 mlrun/feature_store/common.py,sha256=Z7USI-d1fo0iwBMsqMBtJflJfyuiV3BLoDXQPSAoBAs,12826
@@ -217,20 +217,21 @@ mlrun/launcher/factory.py,sha256=RW7mfzEFi8fR0M-4W1JQg1iq3_muUU6OTqT_3l4Ubrk,233
 mlrun/launcher/local.py,sha256=775HY-8S9LFUX5ubGXrLO0N1lVh8bn-DHFmNYuNqQPA,11451
 mlrun/launcher/remote.py,sha256=rLJW4UAnUT5iUb4BsGBOAV3K4R29a0X4lFtRkVKlyYU,7709
 mlrun/model_monitoring/__init__.py,sha256=ELy7njEtZnz09Dc6PGZSFFEGtnwI15bJNWM3Pj4_YIs,753
-mlrun/model_monitoring/api.py,sha256=3QoMEmJ523rzoWFRkx6SmZ9s0Y5b3RX8bZZMHUoZnf0,28484
+mlrun/model_monitoring/api.py,sha256=w6jjrWhlptm7MruzWIJOqCSi4W_zZSq5kkUgTjvqrOk,28508
 mlrun/model_monitoring/controller.py,sha256=j6hqNYKhrw37PJZBcW4BgjsCpG7PtVMvFTpnZO95QVQ,29078
 mlrun/model_monitoring/features_drift_table.py,sha256=c6GpKtpOJbuT1u5uMWDL_S-6N4YPOmlktWMqPme3KFY,25308
-mlrun/model_monitoring/helpers.py,sha256=rsWH_u0qPRwS9URYI1yK8hUeQc7jhJ2alvt_RLz63oU,22402
+mlrun/model_monitoring/helpers.py,sha256=fx2mCQkDu_PgO9LT7ykJ3qcZ7BwELaPqCt_MejqeVxo,22450
 mlrun/model_monitoring/stream_processing.py,sha256=NcvUdfVzveFzmphU65sFGfxp9Jh5ZKq2tSiWWftML9A,34531
 mlrun/model_monitoring/tracking_policy.py,sha256=PBIGrUYWrwcE5gwXupBIVzOb0QRRwPJsgQm_yLGQxB4,5595
 mlrun/model_monitoring/writer.py,sha256=vbL7bqTyNu8q4bNcebX72sUMybVDAoTWg-CXq4fov3Y,8429
-mlrun/model_monitoring/applications/__init__.py,sha256=QYvzgCutFdAkzqKPD3mvkX_3c1X4tzd-kW8ojUOE9ic,889
+mlrun/model_monitoring/applications/__init__.py,sha256=xDBxkBjl-whHSG_4t1mLkxiypLH-fzn8TmAW9Mjo2uI,759
 mlrun/model_monitoring/applications/_application_steps.py,sha256=97taCEkfGx-QO-gD9uKnRF1PDIxQhY7sjPg85GxgIpA,6628
-mlrun/model_monitoring/applications/base.py,sha256=9mRnykWqWnKrtJ6qrov1t6sjEz56YaID9_eh_5Cq7EA,21334
+mlrun/model_monitoring/applications/base.py,sha256=1jdZAreSqugcVd2xD0-dT6DAt16MO7WKQEK71G6Q6qs,24067
 mlrun/model_monitoring/applications/context.py,sha256=xqbKS61iXE6jBekyW8zjo_E3lxe2D8VepuXG_BA5y2k,14931
-mlrun/model_monitoring/applications/evidently_base.py,sha256=hRjXuXf6xf8sbjGt9yYfGDUGnvS5rV3W7tkJroF3QJA,5098
 mlrun/model_monitoring/applications/histogram_data_drift.py,sha256=G26_4gQfcwDZe3S6SIZ4Uc_qyrHAJ6lDTFOQGkbfQR8,14455
 mlrun/model_monitoring/applications/results.py,sha256=_qmj6TWT0SR2bi7gUyRKBU418eGgGoLW2_hTJ7S-ock,5782
+mlrun/model_monitoring/applications/evidently/__init__.py,sha256=-DqdPnBSrjZhFvKOu_Ie3MiFvlur9sPTZpZ1u0_1AE8,690
+mlrun/model_monitoring/applications/evidently/base.py,sha256=hRjXuXf6xf8sbjGt9yYfGDUGnvS5rV3W7tkJroF3QJA,5098
 mlrun/model_monitoring/db/__init__.py,sha256=r47xPGZpIfMuv8J3PQCZTSqVPMhUta4sSJCZFKcS7FM,644
 mlrun/model_monitoring/db/_schedules.py,sha256=AKyCJBAt0opNE3K3pg2TjCoD_afk1LKw5TY88rLQ2VA,6097
 mlrun/model_monitoring/db/_stats.py,sha256=VVMWLMqG3Us3ozBkLaokJF22Ewv8WKmVE1-OvS_g9vA,6943
@@ -240,7 +241,7 @@ mlrun/model_monitoring/db/tsdb/helpers.py,sha256=0oUXc4aUkYtP2SGP6jTb3uPPKImIUsV
 mlrun/model_monitoring/db/tsdb/tdengine/__init__.py,sha256=vgBdsKaXUURKqIf3M0y4sRatmSVA4CQiJs7J5dcVBkQ,620
 mlrun/model_monitoring/db/tsdb/tdengine/schemas.py,sha256=qfKDUZhgteL0mp2A1aP1iMmcthgUMKmZqMUidZjQktQ,12649
 mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py,sha256=Uadj0UvAmln2MxDWod-kAzau1uNlqZh981rPhbUH_5M,2857
-mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=mFvajKtmZ7EHre7b1px0QdnvHBjcipLr6PrKZekjSwY,32716
+mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=OgIA-S-DZ3DM455zOk24rUsmM2EdrKbliuKs1Lzcpuw,32793
 mlrun/model_monitoring/db/tsdb/v3io/__init__.py,sha256=aL3bfmQsUQ-sbvKGdNihFj8gLCK3mSys0qDcXtYOwgc,616
 mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py,sha256=_-zo9relCDtjGgievxAcAP9gVN9nDWs8BzGtFwTjb9M,6284
 mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=foxYWx7OjOfat2SHmzYrG8bIfaQ5NDnBtpDZua_NVGE,41141
@@ -267,23 +268,23 @@ mlrun/platforms/__init__.py,sha256=ZuyeHCHHUxYEoZRmaJqzFSfwhaTyUdBZXMeVp75ql1w,3
 mlrun/platforms/iguazio.py,sha256=6VBTq8eQ3mzT96tzjYhAtcMQ2VjF4x8LpIPW5DAcX2Q,13749
 mlrun/projects/__init__.py,sha256=0Krf0WIKfnZa71WthYOg0SoaTodGg3sV_hK3f_OlTPI,1220
 mlrun/projects/operations.py,sha256=VXUlMrouFTls-I-bMhdN5pPfQ34TR7bFQ-NUSWNvl84,20029
-mlrun/projects/pipelines.py,sha256=iSFSklUqhhR_0fvnG9AS4jWgt8kR3NrKo7vW9ehX4uo,48259
-mlrun/projects/project.py,sha256=f1tNPVc_jqeXV0BnyYHEvQC1PZSQOydPvop-6_NlJBg,234476
+mlrun/projects/pipelines.py,sha256=QH2nEhaJxhafJdT0AXPzpDhTniyHtc0Cg74Spdz6Oeg,48255
+mlrun/projects/project.py,sha256=89fKq2kdfV1mtMMj0EIT55eI_F3IUKk_dCeAk7S-5AA,234506
 mlrun/runtimes/__init__.py,sha256=J9Sy2HiyMlztNv6VUurMzF5H2XzttNil8nRsWDsqLyg,8923
-mlrun/runtimes/base.py,sha256=aAEGZKPkcFs30UzURS7al3xYEDDARpJQ8kFhtBKUhik,37845
+mlrun/runtimes/base.py,sha256=K5-zfFrE_HR6AaHWs2figaOTr7eosw3-4bELkYzpRk4,37789
 mlrun/runtimes/daskjob.py,sha256=JwuGvOiPsxEDHHMMUS4Oie4hLlYYIZwihAl6DjroTY0,19521
 mlrun/runtimes/funcdoc.py,sha256=zRFHrJsV8rhDLJwoUhcfZ7Cs0j-tQ76DxwUqdXV_Wyc,9810
-mlrun/runtimes/function_reference.py,sha256=iWKRe4r2GTc5S8FOIASYUNLwwne8NqIui51PFr8Q4mg,4918
+mlrun/runtimes/function_reference.py,sha256=CLvRY-wXX9qhI9YEzSl0VWt8piH_-5FQYQ8ObUYLLDc,4911
 mlrun/runtimes/generators.py,sha256=X8NDlCEPveDDPOHtOGcSpbl3pAVM3DP7fuPj5xVhxEY,7290
 mlrun/runtimes/kubejob.py,sha256=gJnlAJ0RJw65yeiIPuLEjxJkDYfbpRgS3lyWkDDFXTk,8797
 mlrun/runtimes/local.py,sha256=yedo3R1c46cB1mX7aOz8zORXswQPvX86U-_fYxXoqTY,22717
 mlrun/runtimes/mounts.py,sha256=pGQlnsNTUxAhFMWLS_53E784z-IH9a6oQjKjSp1gbJE,18733
-mlrun/runtimes/pod.py,sha256=VsxviESdIW9eMM4XUnpIQk8OlRiCEUbZpI68Mmwy5Ro,67708
+mlrun/runtimes/pod.py,sha256=hOIXw6X5y7Vkzd6lvt-2reeh2ockj3Bf7xVBUn6xGeo,71824
 mlrun/runtimes/remotesparkjob.py,sha256=dod99nqz3GdRfmnBoQKfwFCXTetfuCScd2pKH3HJyoY,7394
 mlrun/runtimes/utils.py,sha256=3_Vu_OHlhi8f0vh_w9ii2eTKgS5dh6RVi1HwX9oDKuU,15675
 mlrun/runtimes/databricks_job/__init__.py,sha256=kXGBqhLN0rlAx0kTXhozGzFsIdSqW0uTSKMmsLgq_is,569
 mlrun/runtimes/databricks_job/databricks_cancel_task.py,sha256=sIqIg5DQAf4j0wCPA-G0GoxY6vacRddxCy5KDUZszek,2245
-mlrun/runtimes/databricks_job/databricks_runtime.py,sha256=p80j2_jHzlH20dHT-avjfcbaDBTY2re1WjlJjbg5uSQ,12794
+mlrun/runtimes/databricks_job/databricks_runtime.py,sha256=THzAuVMdGWeqTiXGj6Dy0d8SZpHbUZdDKvoxKYO5fTM,12816
 mlrun/runtimes/databricks_job/databricks_wrapper.py,sha256=oJzym54jD957yzxRXiSYpituSV8JV_XJh90YTKIwapY,8684
 mlrun/runtimes/mpijob/__init__.py,sha256=6sUPQRFwigi4mqjDVZmRE-qgaLw2ILY5NbneVUuMKto,947
 mlrun/runtimes/mpijob/abstract.py,sha256=JGMjcJ4dvpJbctF6psU9UvYyNCutMxTMgBQeTlzpkro,9249
@@ -319,7 +320,7 @@ mlrun/utils/azure_vault.py,sha256=IEFizrDGDbAaoWwDr1WoA88S_EZ0T--vjYtY-i0cvYQ,34
 mlrun/utils/clones.py,sha256=y3zC9QS7z5mLuvyQ6vFd6sJnikbgtDwrBvieQq0sovY,7359
 mlrun/utils/condition_evaluator.py,sha256=-nGfRmZzivn01rHTroiGY4rqEv8T1irMyhzxEei-sKc,1897
 mlrun/utils/db.py,sha256=blQgkWMfFH9lcN4sgJQcPQgEETz2Dl_zwbVA0SslpFg,2186
-mlrun/utils/helpers.py,sha256=kaYKV1HoH-DGYSMPEZ_x1yc-1YtU6auGkx4I48XbnbU,72230
+mlrun/utils/helpers.py,sha256=adXqZYrCWXhbNOGI4J1K4WBXsSAIkv-FWaNayt424xs,73279
 mlrun/utils/http.py,sha256=t6FrXQstZm9xVVjxqIGiLzrwZNCR4CSienSOuVgNIcI,8706
 mlrun/utils/logger.py,sha256=RG0m1rx6gfkJ-2C1r_p41MMpPiaDYqaYM2lYHDlNZEU,14767
 mlrun/utils/regex.py,sha256=jbR7IiOp6OO0mg9Fl_cVZCpWb9fL9nTPONCUxCDNWXg,5201
@@ -338,11 +339,11 @@ mlrun/utils/notifications/notification/mail.py,sha256=ZyJ3eqd8simxffQmXzqd3bgbAq
 mlrun/utils/notifications/notification/slack.py,sha256=eQvmctTh6wIG5xVOesLLV9S1-UUCu5UEQ9JIJOor3ts,7183
 mlrun/utils/notifications/notification/webhook.py,sha256=NeyIMSBojjjTJaUHmPbxMByp34GxYkl1-16NqzU27fU,4943
 mlrun/utils/version/__init__.py,sha256=7kkrB7hEZ3cLXoWj1kPoDwo4MaswsI2JVOBpbKgPAgc,614
-mlrun/utils/version/version.json,sha256=SaPAMx9h5srPLn9C27u30d_vVEwOOn2ZKxNhEF1-k-k,89
+mlrun/utils/version/version.json,sha256=HiW1MAPj8GSuC6bLGzBX2dEOLzDlgCTLe3iWUM7fm1s,89
 mlrun/utils/version/version.py,sha256=eEW0tqIAkU9Xifxv8Z9_qsYnNhn3YH7NRAfM-pPLt1g,1878
-mlrun-1.8.0rc31.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mlrun-1.8.0rc31.dist-info/METADATA,sha256=Dm4rCBBu0WElji39X7P0nCB0-HEGwzx4vLKY7UUWeiM,25985
-mlrun-1.8.0rc31.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-mlrun-1.8.0rc31.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
-mlrun-1.8.0rc31.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.8.0rc31.dist-info/RECORD,,
+mlrun-1.8.0rc33.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mlrun-1.8.0rc33.dist-info/METADATA,sha256=CKvUMuVAhnREX815xtMEgz1TMTjllhP0ywgeXRFctiw,25986
+mlrun-1.8.0rc33.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+mlrun-1.8.0rc33.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
+mlrun-1.8.0rc33.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.8.0rc33.dist-info/RECORD,,