mlrun 1.8.0rc62__py3-none-any.whl → 1.9.0__py3-none-any.whl

mlrun/__main__.py

@@ -13,6 +13,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import functools
+import importlib.metadata
 import json
 import pathlib
 import socket
@@ -25,12 +27,14 @@ from pprint import pprint
 import click
 import dotenv
 import pandas as pd
+import semver
 import yaml
 from tabulate import tabulate
 
 import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
+import mlrun.platforms
 import mlrun.utils.helpers
 from mlrun.common.helpers import parse_versioned_object_uri
 from mlrun.runtimes.mounts import auto_mount as auto_mount_modifier
@@ -63,16 +67,22 @@ from .utils.version import Version
 pd.set_option("mode.chained_assignment", None)
 
 
-def validate_base_argument(ctx, param, value):
+def validate_base_argument(ctx: click.Context, param: click.Parameter, value: str):
+    # click 8.2 expects the context to be passed to make_metavar
+    if semver.VersionInfo.parse(
+        importlib.metadata.version("click")
+    ) < semver.VersionInfo.parse("8.2.0"):
+        metavar_func = functools.partial(param.make_metavar)
+    else:
+        metavar_func = functools.partial(param.make_metavar, ctx)
     if value and value.startswith("-"):
         raise click.BadParameter(
             f"{param.human_readable_name} ({value}) cannot start with '-', ensure the command options are typed "
             f"correctly. Preferably use '--' to separate options and arguments "
-            f"e.g. 'mlrun run --option1 --option2 -- {param.make_metavar()} [--arg1|arg1] [--arg2|arg2]'",
+            f"e.g. 'mlrun run --option1 --option2 -- {metavar_func()} [--arg1|arg1] [--arg2|arg2]'",
             ctx=ctx,
             param=param,
         )
-
     return value
 
 

mlrun/common/constants.py

@@ -90,6 +90,13 @@ class MLRunInternalLabels:
             if not key.startswith("__") and isinstance(value, str)
         ]
 
+    @staticmethod
+    def default_run_labels_to_enrich():
+        return [
+            MLRunInternalLabels.owner,
+            MLRunInternalLabels.v3io_user,
+        ]
+
 
 class DeployStatusTextKind(mlrun.common.types.StrEnum):
     logs = "logs"

mlrun/config.py

@@ -64,7 +64,7 @@ default_config = {
     # url to nuclio dashboard api (can be with user & token, e.g. https://username:password@dashboard-url.com)
     "nuclio_dashboard_url": "",
     "nuclio_version": "",
-    "default_nuclio_runtime": "python:3.9",
+    "default_nuclio_runtime": "python:3.11",
     "nest_asyncio_enabled": "",  # enable import of nest_asyncio for corner cases with old jupyter, set "1"
     "ui_url": "",  # remote/external mlrun UI url (for hyperlinks) (This is deprecated in favor of the ui block)
     "remote_host": "",
@@ -79,7 +79,7 @@ default_config = {
     # comma separated list of images that are in the specified images_registry, and therefore will be enriched with this
     # registry when used. default to mlrun/* which means any image which is of the mlrun repository (mlrun/mlrun,
     # mlrun/ml-base, etc...)
-    "images_to_enrich_registry": "^mlrun/*,python:3.9",
+    "images_to_enrich_registry": "^mlrun/*,^python:3.(9|11)$",
     "kfp_url": "",
     "kfp_ttl": "14400",  # KFP ttl in sec, after that completed PODs will be deleted
     "kfp_image": "mlrun/mlrun-kfp",  # image to use for KFP runner
@@ -286,7 +286,7 @@ default_config = {
             "remote": "mlrun/mlrun",
             "dask": "mlrun/ml-base",
             "mpijob": "mlrun/mlrun",
-            "application": "python:3.9",
+            "application": "python",
         },
         # see enrich_function_preemption_spec for more info,
         # and mlrun.common.schemas.function.PreemptionModes for available options

mlrun/datastore/datastore_profile.py

@@ -255,7 +255,7 @@ class DatastoreProfileS3(DatastoreProfile):
     def check_bucket(cls, v):
         if not v:
             warnings.warn(
-                "The 'bucket' attribute will be mandatory starting from version 1.9",
+                "The 'bucket' attribute will be mandatory starting from version 1.10",
                 FutureWarning,
                 stacklevel=2,
             )
@@ -360,7 +360,7 @@ class DatastoreProfileGCS(DatastoreProfile):
     def check_bucket(cls, v):
         if not v:
             warnings.warn(
-                "The 'bucket' attribute will be mandatory starting from version 1.9",
+                "The 'bucket' attribute will be mandatory starting from version 1.10",
                 FutureWarning,
                 stacklevel=2,
             )
@@ -417,7 +417,7 @@ class DatastoreProfileAzureBlob(DatastoreProfile):
     def check_container(cls, v):
         if not v:
             warnings.warn(
-                "The 'container' attribute will be mandatory starting from version 1.9",
+                "The 'container' attribute will be mandatory starting from version 1.10",
                 FutureWarning,
                 stacklevel=2,
             )

mlrun/db/httpdb.py

@@ -21,7 +21,7 @@ import typing
 import warnings
 from copy import deepcopy
 from datetime import datetime, timedelta
-from os import path, remove
+from os import environ, path, remove
 from typing import Literal, Optional, Union
 from urllib.parse import urlparse
 
@@ -129,7 +129,9 @@ class HTTPRunDB(RunDBInterface):
         self._wait_for_background_task_terminal_state_retry_interval = 3
         self._wait_for_project_deletion_interval = 3
         self.client_version = version.Version().get()["version"]
-        self.python_version = str(version.Version().get_python_version())
+        self.python_version = environ.get("MLRUN_PYTHON_VERSION") or str(
+            version.Version().get_python_version()
+        )
 
         self._enrich_and_validate(url)
 

mlrun/frameworks/tf_keras/__init__.py

@@ -14,7 +14,7 @@
 
 from typing import Any, Optional, Union
 
-from tensorflow import keras
+import tensorflow as tf
 
 import mlrun
 import mlrun.common.constants as mlrun_constants
@@ -27,11 +27,11 @@ from .utils import TFKerasTypes, TFKerasUtils
 
 
 def apply_mlrun(
-    model: keras.Model = None,
+    model: tf.keras.Model = None,
     model_name: Optional[str] = None,
     tag: str = "",
     model_path: Optional[str] = None,
-    model_format: str = TFKerasModelHandler.ModelFormats.SAVED_MODEL,
+    model_format: Optional[str] = None,
     save_traces: bool = False,
     modules_map: Optional[Union[dict[str, Union[None, str, list[str]]], str]] = None,
     custom_objects_map: Optional[Union[dict[str, Union[str, list[str]]], str]] = None,
@@ -54,7 +54,7 @@ def apply_mlrun(
     :param model_path:                  The model's store object path. Mandatory for evaluation (to know which model to
                                         update). If model is not provided, it will be loaded from this path.
     :param model_format:                The format to use for saving and loading the model. Should be passed as a
-                                        member of the class 'ModelFormats'. Default: 'ModelFormats.SAVED_MODEL'.
+                                        member of the class 'ModelFormats'.
     :param save_traces:                 Whether or not to use functions saving (only available for the 'SavedModel'
                                         format) for loading the model later without the custom objects dictionary. Only
                                         from tensorflow version >= 2.4.0. Using this setting will increase the model

mlrun/frameworks/tf_keras/callbacks/logging_callback.py

@@ -16,14 +16,14 @@ from typing import Callable, Optional, Union
 
 import numpy as np
 import tensorflow as tf
-from tensorflow import Tensor, Variable
+from tensorflow import keras
 from tensorflow.python.keras.callbacks import Callback
 
 import mlrun
 
 from ..._common import LoggingMode
 from ..._dl_common.loggers import Logger
-from ..utils import TFKerasTypes
+from ..utils import TFKerasTypes, is_keras_3
 
 
 class LoggingCallback(Callback):
@@ -70,7 +70,7 @@ class LoggingCallback(Callback):
                                         {
                                             "epochs": 7
                                         }
-        :param auto_log:                Whether or not to enable auto logging, trying to track common static and dynamic
+        :param auto_log:                Whether to enable auto logging, trying to track common static and dynamic
                                         hyperparameters.
         """
         super().__init__()
@@ -385,18 +385,24 @@ class LoggingCallback(Callback):
             self._logger.log_context_parameters()
 
         # Add learning rate:
-        learning_rate_key = "lr"
-        learning_rate_key_chain = ["optimizer", "lr"]
-        if learning_rate_key not in self._dynamic_hyperparameters_keys and hasattr(
-            self.model, "optimizer"
-        ):
-            try:
-                self._get_hyperparameter(key_chain=learning_rate_key_chain)
+        learning_rate_keys = [
+            "learning_rate",
+            "lr",
+        ]  # "lr" is for backward compatibility in older keras versions.
+        if all(
+            learning_rate_key not in self._dynamic_hyperparameters_keys
+            for learning_rate_key in learning_rate_keys
+        ) and hasattr(self.model, "optimizer"):
+            for learning_rate_key in learning_rate_keys:
+                learning_rate_key_chain = ["optimizer", learning_rate_key]
+                try:
+                    self._get_hyperparameter(key_chain=learning_rate_key_chain)
+                except (KeyError, IndexError, AttributeError, ValueError):
+                    continue
                 self._dynamic_hyperparameters_keys[learning_rate_key] = (
                     learning_rate_key_chain
                 )
-            except (KeyError, IndexError, ValueError):
-                pass
+                break
 
     def _get_hyperparameter(
         self,
@@ -427,7 +433,7 @@ class LoggingCallback(Callback):
                         value = value[key]
                     else:
                         value = getattr(value, key)
-                except KeyError or IndexError as KeyChainError:
+                except KeyError or IndexError or AttributeError as KeyChainError:
                     raise KeyChainError(
                         f"Error during getting a hyperparameter value with the key chain {key_chain}. "
                         f"The {value.__class__} in it does not have the following key/index from the key provided: "
@@ -435,7 +441,9 @@ class LoggingCallback(Callback):
                     )
 
         # Parse the value:
-        if isinstance(value, Tensor) or isinstance(value, Variable):
+        if isinstance(value, (tf.Tensor, tf.Variable)) or (
+            is_keras_3() and isinstance(value, (keras.KerasTensor, keras.Variable))
+        ):
             if int(tf.size(value)) == 1:
                 value = float(value)
             else:
@@ -451,12 +459,7 @@ class LoggingCallback(Callback):
                     f"The parameter with the following key chain: {key_chain} is a numpy.ndarray with {value.size} "
                     f"elements. numpy arrays are trackable only if they have 1 element."
                 )
-        elif not (
-            isinstance(value, float)
-            or isinstance(value, int)
-            or isinstance(value, str)
-            or isinstance(value, bool)
-        ):
+        elif not (isinstance(value, (float, int, str, bool))):
             raise mlrun.errors.MLRunInvalidArgumentError(
                 f"The parameter with the following key chain: {key_chain} is of type '{type(value)}'. The only "
                 f"trackable types are: float, int, str and bool."

mlrun/frameworks/tf_keras/model_handler.py

@@ -29,7 +29,7 @@ from mlrun.features import Feature
 from .._common import without_mlrun_interface
 from .._dl_common import DLModelHandler
 from .mlrun_interface import TFKerasMLRunInterface
-from .utils import TFKerasUtils
+from .utils import TFKerasUtils, is_keras_3
 
 
 class TFKerasModelHandler(DLModelHandler):
@@ -40,8 +40,8 @@ class TFKerasModelHandler(DLModelHandler):
     # Framework name:
     FRAMEWORK_NAME = "tensorflow.keras"
 
-    # Declare a type of an input sample:
-    IOSample = Union[tf.Tensor, tf.TensorSpec, np.ndarray]
+    # Declare a type of input sample (only from keras v3 there is a KerasTensor type):
+    IOSample = Union[tf.Tensor, tf.TensorSpec, "keras.KerasTensor", np.ndarray]
 
     class ModelFormats:
         """
@@ -49,9 +49,19 @@ class TFKerasModelHandler(DLModelHandler):
         """
 
         SAVED_MODEL = "SavedModel"
+        KERAS = "keras"
         H5 = "h5"
         JSON_ARCHITECTURE_H5_WEIGHTS = "json_h5"
 
+        @classmethod
+        def default(cls) -> str:
+            """
+            Get the default model format to use for saving and loading the model based on the keras version.
+
+            :return: The default model format to use.
+            """
+            return cls.KERAS if is_keras_3() else cls.SAVED_MODEL
+
     class _LabelKeys:
         """
         Required labels keys to log with the model.
@@ -65,7 +75,7 @@ class TFKerasModelHandler(DLModelHandler):
         model: keras.Model = None,
         model_path: Optional[str] = None,
         model_name: Optional[str] = None,
-        model_format: str = ModelFormats.SAVED_MODEL,
+        model_format: Optional[str] = None,
         context: mlrun.MLClientCtx = None,
         modules_map: Optional[
             Union[dict[str, Union[None, str, list[str]]], str]
@@ -98,7 +108,7 @@ class TFKerasModelHandler(DLModelHandler):
                                          * If given a loaded model object and the model name is None, the name will be
                                            set to the model's object name / class.
         :param model_format:             The format to use for saving and loading the model. Should be passed as a
-                                         member of the class 'ModelFormats'. Default: 'ModelFormats.SAVED_MODEL'.
+                                         member of the class 'ModelFormats'.
         :param context:                  MLRun context to work with for logging the model.
         :param modules_map:              A dictionary of all the modules required for loading the model. Each key
                                          is a path to a module and its value is the object name to import from it. All
@@ -144,8 +154,11 @@ class TFKerasModelHandler(DLModelHandler):
                                           * 'save_traces' parameter was miss-used.
         """
         # Validate given format:
+        if not model_format:
+            model_format = TFKerasModelHandler.ModelFormats.default()
         if model_format not in [
             TFKerasModelHandler.ModelFormats.SAVED_MODEL,
+            TFKerasModelHandler.ModelFormats.KERAS,
             TFKerasModelHandler.ModelFormats.H5,
             TFKerasModelHandler.ModelFormats.JSON_ARCHITECTURE_H5_WEIGHTS,
         ]:
@@ -153,6 +166,22 @@ class TFKerasModelHandler(DLModelHandler):
                 f"Unrecognized model format: '{model_format}'. Please use one of the class members of "
                 "'TFKerasModelHandler.ModelFormats'"
             )
+        if not is_keras_3():
+            if model_format == TFKerasModelHandler.ModelFormats.KERAS:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "The 'keras' model format is only supported in Keras 3.0.0 and above. "
+                    f"Current version is {keras.__version__}."
+                )
+        else:
+            if (
+                model_format == TFKerasModelHandler.ModelFormats.SAVED_MODEL
+                or model_format
+                == TFKerasModelHandler.ModelFormats.JSON_ARCHITECTURE_H5_WEIGHTS
+            ):
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    f"The '{model_format}' model format is not supported in Keras 3.0.0 and above. "
+                    f"Current version is {keras.__version__}."
+                )
 
         # Validate 'save_traces':
         if save_traces:
@@ -239,11 +268,19 @@ class TFKerasModelHandler(DLModelHandler):
             self._model_file = f"{self._model_name}.h5"
             self._model.save(self._model_file)
 
+        # ModelFormats.keras - Save as a keras file:
+        elif self._model_format == self.ModelFormats.KERAS:
+            self._model_file = f"{self._model_name}.keras"
+            self._model.save(self._model_file)
+
         # ModelFormats.SAVED_MODEL - Save as a SavedModel directory and zip its file:
         elif self._model_format == TFKerasModelHandler.ModelFormats.SAVED_MODEL:
             # Save it in a SavedModel format directory:
+            # Note: Using keras>=3.0.0 can save in this format via `model.export` but then it won't be able to load it
+            # back, only for inference. So, we use the `save` method instead for keras 2 and validate the user won't use
+            # keras 3 and this model format.
             if self._save_traces is True:
-                # Save traces can only be used in versions >= 2.4, so only if its true we use it in the call:
+                # Save traces can only be used in versions >= 2.4, so only if it's true, we use it in the call:
                 self._model.save(self._model_name, save_traces=self._save_traces)
             else:
                 self._model.save(self._model_name)
@@ -303,6 +340,12 @@ class TFKerasModelHandler(DLModelHandler):
                 self._model_file, custom_objects=self._custom_objects
             )
 
+        # ModelFormats.KERAS - Load from a keras file:
+        elif self._model_format == TFKerasModelHandler.ModelFormats.KERAS:
+            self._model = keras.models.load_model(
+                self._model_file, custom_objects=self._custom_objects
+            )
+
         # ModelFormats.SAVED_MODEL - Load from a SavedModel directory:
         elif self._model_format == TFKerasModelHandler.ModelFormats.SAVED_MODEL:
             self._model = keras.models.load_model(
@@ -434,7 +477,10 @@ class TFKerasModelHandler(DLModelHandler):
             )
 
         # Read the inputs:
-        input_signature = [input_layer.type_spec for input_layer in self._model.inputs]
+        input_signature = [
+            getattr(input_layer, "type_spec", input_layer)
+            for input_layer in self._model.inputs
+        ]
 
         # Set the inputs:
         self.set_inputs(from_sample=input_signature)
@@ -453,7 +499,8 @@ class TFKerasModelHandler(DLModelHandler):
 
         # Read the outputs:
         output_signature = [
-            output_layer.type_spec for output_layer in self._model.outputs
+            getattr(output_layer, "type_spec", output_layer)
+            for output_layer in self._model.outputs
         ]
 
         # Set the outputs:
@@ -509,6 +556,17 @@ class TFKerasModelHandler(DLModelHandler):
                     f"'{self._model_path}'"
                 )
 
+        # ModelFormats.KERAS - Get the keras model file:
+        elif self._model_format == TFKerasModelHandler.ModelFormats.KERAS:
+            self._model_file = os.path.join(
+                self._model_path, f"{self._model_name}.keras"
+            )
+            if not os.path.exists(self._model_file):
+                raise mlrun.errors.MLRunNotFoundError(
+                    f"The model file '{self._model_name}.keras' was not found within the given 'model_path': "
+                    f"'{self._model_path}'"
+                )
+
         # ModelFormats.SAVED_MODEL - Get the zip file and extract it, or simply locate the directory:
         elif self._model_format == TFKerasModelHandler.ModelFormats.SAVED_MODEL:
             self._model_file = os.path.join(self._model_path, f"{self._model_name}.zip")
@@ -559,7 +617,9 @@ class TFKerasModelHandler(DLModelHandler):
         # Supported types:
         if isinstance(sample, np.ndarray):
             return super()._read_sample(sample=sample)
-        elif isinstance(sample, tf.TensorSpec):
+        elif isinstance(sample, tf.TensorSpec) or (
+            is_keras_3() and isinstance(sample, keras.KerasTensor)
+        ):
             return Feature(
                 name=sample.name,
                 value_type=TFKerasUtils.convert_tf_dtype_to_value_type(

mlrun/frameworks/tf_keras/utils.py

@@ -11,8 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 import tensorflow as tf
+from packaging import version
 from tensorflow import keras
 
 import mlrun
@@ -117,3 +117,14 @@ class TFKerasUtils(DLUtils):
         raise mlrun.errors.MLRunInvalidArgumentError(
             f"MLRun value type is not supporting the given tensorflow data type: '{tf_dtype}'."
         )
+
+
+def is_keras_3() -> bool:
+    """
+    Check if the current Keras version is 3.x.
+
+    :return: True if Keras version is 3.x, False otherwise.
+    """
+    return hasattr(keras, "__version__") and version.parse(
+        keras.__version__
+    ) >= version.parse("3.0.0")

mlrun/launcher/client.py

@@ -72,7 +72,7 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
     ):
         run.metadata.labels[mlrun_constants.MLRunInternalLabels.kind] = runtime.kind
         mlrun.runtimes.utils.enrich_run_labels(
-            run.metadata.labels, [mlrun.common.runtimes.constants.RunLabels.owner]
+            run.metadata.labels, [mlrun_constants.MLRunInternalLabels.owner]
         )
         if run.spec.output_path:
             run.spec.output_path = run.spec.output_path.replace(

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connection.py

@@ -11,8 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-import traceback
+import time
 from collections.abc import Callable
 from enum import Enum
 from typing import Any, Final, Optional, Union
@@ -20,6 +19,9 @@ from typing import Any, Final, Optional, Union
 import taosws
 from taosws import TaosStmt
 
+import mlrun
+from mlrun.utils import logger
+
 
 class _StrEnum(str, Enum):
     pass
@@ -137,40 +139,99 @@ class Statement:
         return statement
 
 
-def _run(connection_string, prefix_statements, q, statements, query):
-    try:
-        conn = taosws.connect(connection_string)
-
-        for statement in prefix_statements + statements:
-            if isinstance(statement, Statement):
-                prepared_statement = statement.prepare(conn.statement())
-                prepared_statement.execute()
-            else:
-                conn.execute(statement)
-
-        if not query:
-            q.put(None)
-            return
-
-        res = conn.query(query)
-
-        # taosws.TaosField is not serializable
-        fields = [
-            Field(field.name(), field.type(), field.bytes()) for field in res.fields
-        ]
-
-        q.put(QueryResult(list(res), fields))
-    except Exception as e:
-        tb = traceback.format_exc()
-        q.put(ErrorResult(tb, e))
-
-
 class TDEngineConnection:
-    def __init__(self, connection_string):
+    def __init__(self, connection_string, max_retries=3, retry_delay=0.5):
         self._connection_string = connection_string
         self.prefix_statements = []
+        self._max_retries = max_retries
+        self._retry_delay = retry_delay
 
-        self._conn = taosws.connect(self._connection_string)
+        self._conn = self._create_connection()
+
+    def _create_connection(self):
+        """Create a new TDEngine connection."""
+        return taosws.connect(self._connection_string)
+
+    def _reconnect(self):
+        """Close current connection and create a new one."""
+        try:
+            if hasattr(self, "_conn") and self._conn:
+                self._conn.close()
+        except Exception as e:
+            logger.warning(f"Error closing connection during reconnect: {e}")
+
+        self._conn = self._create_connection()
+        logger.info("Successfully reconnected to TDEngine")
+
+    def _execute_with_retry(self, operation, operation_name, *args, **kwargs):
+        """
+        Execute an operation with retry logic for connection failures.
+
+        :param operation: The function to execute
+        :param operation_name: Name of the operation for logging
+        :param args: Arguments to pass to the operation
+        :param kwargs: Keyword arguments to pass to the operation
+        :return: Result of the operation
+        """
+        last_exception = None
+
+        for attempt in range(self._max_retries + 1):  # +1 for initial attempt
+            try:
+                return operation(*args, **kwargs)
+
+            except taosws.Error as e:
+                last_exception = e
+
+                if attempt < self._max_retries:
+                    logger.warning(
+                        f"Connection error during {operation_name} "
+                        f"(attempt {attempt + 1}/{self._max_retries + 1}): {e}. "
+                        f"Retrying in {self._retry_delay} seconds..."
+                    )
+
+                    # Wait before retrying
+                    time.sleep(self._retry_delay)
+
+                    # Reconnect
+                    try:
+                        self._reconnect()
+                    except Exception as reconnect_error:
+                        logger.error(f"Failed to reconnect: {reconnect_error}")
+                        if attempt == self._max_retries - 1:
+                            # Last attempt, raise the reconnection error
+                            raise TDEngineError(
+                                f"Failed to reconnect after {operation_name} failure: {reconnect_error}"
+                            ) from reconnect_error
+                        continue
+                else:
+                    # Max retries exceeded
+                    logger.error(
+                        f"Max retries ({self._max_retries}) exceeded for {operation_name}"
+                    )
+                    break
+
+            except Exception as e:
+                # Non-TDEngine error, don't retry
+                raise TDEngineError(
+                    f"Unexpected error during {operation_name}: {e}"
+                ) from e
+
+        # If we get here, all retries failed
+        raise TDEngineError(
+            f"Failed to {operation_name} after {self._max_retries} retries: {last_exception}"
+        ) from last_exception
+
+    def _execute_statement(self, statement):
+        """Execute a single statement (string or Statement object)."""
+        if isinstance(statement, Statement):
+            prepared_statement = statement.prepare(self._conn.statement())
+            prepared_statement.execute()
+        else:
+            self._conn.execute(statement)
+
+    def _execute_query(self, query):
+        """Execute a query and return the result."""
+        return self._conn.query(query)
 
     def run(
         self,
@@ -181,33 +242,40 @@ class TDEngineConnection:
         if not isinstance(statements, list):
             statements = [statements]
 
-        for statement in self.prefix_statements + statements:
+        # Execute all statements with retry logic
+        all_statements = self.prefix_statements + statements
+        for i, statement in enumerate(all_statements):
+            operation_name = f"execute statement {i + 1}/{len(all_statements)}"
             if isinstance(statement, Statement):
-                try:
-                    prepared_statement = statement.prepare(self._conn.statement())
-                    prepared_statement.execute()
-                except taosws.Error as e:
-                    raise TDEngineError(
-                        f"Failed to run prepared statement `{self._conn.statement()}`: {e}"
-                    ) from e
+                operation_name += " (prepared)"
             else:
-                try:
-                    self._conn.execute(statement)
-                except taosws.Error as e:
-                    raise TDEngineError(
-                        f"Failed to run statement `{statement}`: {e}"
-                    ) from e
+                operation_name += f" `{statement}`"
+
+            self._execute_with_retry(self._execute_statement, operation_name, statement)
 
         if not query:
             return None
 
-        try:
-            res = self._conn.query(query)
-        except taosws.Error as e:
-            raise TDEngineError(f"Failed to run query `{query}`: {e}") from e
+        # Execute query with retry logic
+        res = self._execute_with_retry(
+            self._execute_query, f"execute query `{query}`", query
+        )
 
+        # Process results
         fields = [
             Field(field.name(), field.type(), field.bytes()) for field in res.fields
         ]
 
         return QueryResult(list(res), fields)
+
+    def close(self):
+        """Close the connection."""
+        try:
+            if self._conn:
+                self._conn.close()
+                logger.debug("TDEngine connection closed")
+                self._conn = None
+        except Exception as e:
+            logger.warning(
+                f"Error closing TDEngine connection: {mlrun.errors.err_to_str(e)}"
+            )

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import threading
 from datetime import datetime, timedelta
-from threading import Lock
 from typing import Callable, Final, Literal, Optional, Union
 
 import pandas as pd
@@ -32,8 +32,8 @@ from mlrun.model_monitoring.db.tsdb.tdengine.tdengine_connection import (
 from mlrun.model_monitoring.helpers import get_invocations_fqn
 from mlrun.utils import logger
 
-_connection = None
-_connection_lock = Lock()
+# Thread-local storage for connections
+_thread_local = threading.local()
 
 
 class TDEngineTimestampPrecision(mlrun.common.types.StrEnum):
@@ -76,16 +76,15 @@ class TDEngineConnector(TSDBConnector):
 
     @property
     def connection(self) -> TDEngineConnection:
-        global _connection
-
-        if _connection:
-            return _connection
-
-        with _connection_lock:
-            if not _connection:
-                _connection = self._create_connection()
-
-        return _connection
+        if not hasattr(_thread_local, "connection"):
+            _thread_local.connection = self._create_connection()
+            logger.debug(
+                "Created new TDEngine connection for thread",
+                project=self.project,
+                thread_name=threading.current_thread().name,
+                thread_id=threading.get_ident(),
+            )
+        return _thread_local.connection
 
     def _create_connection(self) -> TDEngineConnection:
         """Establish a connection to the TSDB server."""

mlrun/projects/project.py

@@ -1408,7 +1408,10 @@ class MlrunProject(ModelObj):
                               https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron
                               Note that "local" engine does not support this argument
         :param ttl:           Pipeline ttl in secs (after that the pods will be removed)
-        :param image:         Image for workflow runner job, only for scheduled and remote workflows
+        :param image:         Image for workflow runner job, only for scheduled and remote workflows.
+                              The image must have mlrun[kfp] installed which requires python 3.9.
+                              Therefore, the project default image will not be used for the workflow,
+                              and the image must be specified explicitly.
         :param args:          Argument values (key=value, ..)
         """
 

mlrun/runtimes/base.py

@@ -489,7 +489,7 @@ class BaseRuntime(ModelObj):
     def _store_function(self, runspec, meta, db):
         meta.labels["kind"] = self.kind
         mlrun.runtimes.utils.enrich_run_labels(
-            meta.labels, [mlrun.common.runtimes.constants.RunLabels.owner]
+            meta.labels, [mlrun_constants.MLRunInternalLabels.owner]
         )
         if runspec.spec.output_path:
             runspec.spec.output_path = runspec.spec.output_path.replace(

mlrun/runtimes/utils.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import enum
 import getpass
 import hashlib
 import json
@@ -28,7 +29,6 @@ import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
 import mlrun.utils.regex
 from mlrun.artifacts import TableArtifact
-from mlrun.common.runtimes.constants import RunLabels
 from mlrun.config import config
 from mlrun.errors import err_to_str
 from mlrun.frameworks.parallel_coordinates import gen_pcp_plot
@@ -433,18 +433,35 @@ def enrich_function_from_dict(function, function_dict):
 
 def enrich_run_labels(
     labels: dict,
-    labels_to_enrich: Optional[list[RunLabels]] = None,
+    labels_to_enrich: Optional[list[mlrun_constants.MLRunInternalLabels]] = None,
 ):
+    """
+    Enrich the run labels with the internal labels and the labels enrichment extension
+    :param labels: The run labels dict
+    :param labels_to_enrich: The label keys to enrich from MLRunInternalLabels.default_run_labels_to_enrich
+    :return: The enriched labels dict
+    """
+    # Merge the labels with the labels enrichment extension
     labels_enrichment = {
-        RunLabels.owner: os.environ.get("V3IO_USERNAME") or getpass.getuser(),
+        mlrun_constants.MLRunInternalLabels.owner: os.environ.get("V3IO_USERNAME")
+        or getpass.getuser(),
         # TODO: remove this in 1.10.0
-        RunLabels.v3io_user: os.environ.get("V3IO_USERNAME"),
+        mlrun_constants.MLRunInternalLabels.v3io_user: os.environ.get("V3IO_USERNAME"),
     }
-    labels_to_enrich = labels_to_enrich or RunLabels.all()
+
+    # Resolve which label keys to enrich
+    if labels_to_enrich is None:
+        labels_to_enrich = (
+            mlrun_constants.MLRunInternalLabels.default_run_labels_to_enrich()
+        )
+
+    # Enrich labels
     for label in labels_to_enrich:
+        if isinstance(label, enum.Enum):
+            label = label.value
         enrichment = labels_enrichment.get(label)
-        if label.value not in labels and enrichment:
-            labels[label.value] = enrichment
+        if label not in labels and enrichment:
+            labels[label] = enrichment
     return labels
 
 

mlrun/serving/v2_serving.py

@@ -384,15 +384,15 @@ class V2ModelServer(StepToDict):
         return event
 
     def logged_results(self, request: dict, response: dict, op: str):
-        """hook for controlling which results are tracked by the model monitoring
+        """Hook for controlling which results are tracked by the model monitoring
 
-        this hook allows controlling which input/output data is logged by the model monitoring
-        allow filtering out columns or adding custom values, can also be used to monitor derived metrics
-        for example in image classification calculate and track the RGB values vs the image bitmap
+        This hook allows controlling which input/output data is logged by the model monitoring.
+        It allows filtering out columns or adding custom values, and can also be used to monitor derived metrics,
+        for example in image classification to calculate and track the RGB values vs the image bitmap.
 
-        the request["inputs"] holds a list of input values/arrays, the response["outputs"] holds a list of
-        corresponding output values/arrays (the schema of the input/output fields is stored in the model object),
-        this method should return lists of alternative inputs and outputs which will be monitored
+        The request ["inputs"] holds a list of input values/arrays, the response ["outputs"] holds a list of
+        corresponding output values/arrays (the schema of the input/output fields is stored in the model object).
+        This method should return lists of alternative inputs and outputs which will be monitored.
 
         :param request:   predict/explain request, see model serving docs for details
         :param response:  result from the model predict/explain (after postprocess())
@@ -422,6 +422,7 @@ class V2ModelServer(StepToDict):
 
     def predict(self, request: dict) -> list:
         """model prediction operation
+
         :return: list with the model prediction results (can be multi-port) or list of lists for multiple predictions
         """
         raise NotImplementedError()
@@ -436,7 +437,7 @@ class V2ModelServer(StepToDict):
         where the internal list order is according to the ArtifactModel inputs.
 
         :param request: event
-        :return: evnet body converting the inputs to be list of lists
+        :return: event body converting the inputs to be list of lists
         """
         if self.model_spec and self.model_spec.inputs:
             input_order = [feature.name for feature in self.model_spec.inputs]

mlrun/utils/helpers.py

@@ -876,16 +876,25 @@ def enrich_image_url(
     client_version: Optional[str] = None,
     client_python_version: Optional[str] = None,
 ) -> str:
+    image_url = image_url.strip()
+
+    # Add python version tag if needed
+    if image_url == "python" and client_python_version:
+        image_tag = ".".join(client_python_version.split(".")[:2])
+        image_url = f"python:{image_tag}"
+
     client_version = _convert_python_package_version_to_image_tag(client_version)
     server_version = _convert_python_package_version_to_image_tag(
         mlrun.utils.version.Version().get()["version"]
     )
-    image_url = image_url.strip()
     mlrun_version = config.images_tag or client_version or server_version
-    tag = mlrun_version
-    tag += resolve_image_tag_suffix(
-        mlrun_version=mlrun_version, python_version=client_python_version
-    )
+    tag = mlrun_version or ""
+
+    # TODO: Remove condition when mlrun/mlrun-kfp image is also supported
+    if "mlrun-kfp" not in image_url:
+        tag += resolve_image_tag_suffix(
+            mlrun_version=mlrun_version, python_version=client_python_version
+        )
 
     # it's an mlrun image if the repository is mlrun
     is_mlrun_image = image_url.startswith("mlrun/") or "/mlrun/" in image_url
@@ -917,7 +926,7 @@ def resolve_image_tag_suffix(
     mlrun_version: Optional[str] = None, python_version: Optional[str] = None
 ) -> str:
     """
-    resolves what suffix should be appended to the image tag
+    Resolves what suffix to be appended to the image tag
     :param mlrun_version: the mlrun version
     :param python_version: the requested python version
     :return: the suffix to append to the image tag
@@ -929,19 +938,19 @@ def resolve_image_tag_suffix(
     # mlrun version is higher than 1.3.0, but we can check the python version and if python version was passed it
     # means it 1.3.0-rc or higher, so we can add the suffix of the python version.
     if mlrun_version.startswith("0.0.0-") or "unstable" in mlrun_version:
-        if python_version.startswith("3.7"):
-            return "-py37"
+        if python_version.startswith("3.9"):
+            return "-py39"
         return ""
 
-    # For mlrun 1.3.x and 1.4.x, we support mlrun runtimes images with both python 3.7 and 3.9 images.
-    # While the python 3.9 images will continue to have no suffix, the python 3.7 images will have a '-py37' suffix.
-    # Python 3.8 images will not be supported for mlrun 1.3.0, meaning that if the user has client with python 3.8
-    # and mlrun 1.3.x then the image will be pulled without a suffix (which is the python 3.9 image).
+    # For mlrun 1.9.x and 1.10.x, we support mlrun runtimes images with both python 3.9 and 3.11 images.
+    # While the python 3.11 images will continue to have no suffix, the python 3.9 images will have a '-py39' suffix.
+    # Python 3.10 images are not supported in mlrun 1.9.0, meaning that if the user has client with python 3.10
+    # and mlrun 1.9.x then the image will be pulled without a suffix (which is the python 3.11 image).
     # using semver (x.y.z-X) to include rc versions as well
-    if semver.VersionInfo.parse("1.5.0-X") > semver.VersionInfo.parse(
+    if semver.VersionInfo.parse("1.11.0-X") > semver.VersionInfo.parse(
         mlrun_version
-    ) >= semver.VersionInfo.parse("1.3.0-X") and python_version.startswith("3.7"):
-        return "-py37"
+    ) >= semver.VersionInfo.parse("1.9.0-X") and python_version.startswith("3.9"):
+        return "-py39"
     return ""
 
 
@@ -2254,7 +2263,10 @@ class Workflow:
     def _get_workflow_manifest(
         workflow_id: str,
     ) -> typing.Optional[mlrun_pipelines.models.PipelineManifest]:
-        kfp_client = mlrun_pipelines.utils.get_client(mlrun.mlconf.kfp_url)
+        kfp_client = mlrun_pipelines.utils.get_client(
+            url=mlrun.mlconf.kfp_url,
+            namespace=mlrun.mlconf.namespace,
+        )
 
         # arbitrary timeout of 30 seconds, the workflow should be done by now, however sometimes kfp takes a few
         # seconds to update the workflow status

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "bd76eb04427505ad0944a97ddcf76e50820cce32",
-  "version": "1.8.0-rc62"
+  "git_commit": "f36cbb4f1a9a264471b1fa40a453b382d056988a",
+  "version": "1.9.0"
 }

{mlrun-1.8.0rc62.dist-info → mlrun-1.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.8.0rc62
+Version: 1.9.0
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -15,6 +15,7 @@ Classifier: Operating System :: Microsoft :: Windows
 Classifier: Operating System :: MacOS
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Software Development :: Libraries
@@ -35,7 +36,7 @@ Requires-Dist: pyarrow<17,>=10.0
 Requires-Dist: pyyaml<7,>=6.0.2
 Requires-Dist: requests~=2.32
 Requires-Dist: tabulate~=0.8.6
-Requires-Dist: v3io~=0.6.9
+Requires-Dist: v3io~=0.7.0
 Requires-Dist: pydantic>=1.10.15
 Requires-Dist: mergedeep~=1.3
 Requires-Dist: v3io-frames~=0.10.14; python_version < "3.11"
@@ -44,15 +45,15 @@ Requires-Dist: semver~=3.0
 Requires-Dist: dependency-injector~=4.41
 Requires-Dist: fsspec<2024.7,>=2023.9.2
 Requires-Dist: v3iofs~=0.1.17
-Requires-Dist: storey~=1.8.11
+Requires-Dist: storey~=1.9.0
 Requires-Dist: inflection~=0.5.0
 Requires-Dist: python-dotenv~=1.0
 Requires-Dist: setuptools>=75.2
 Requires-Dist: deprecated~=1.2
 Requires-Dist: jinja2>=3.1.3,~=3.1
 Requires-Dist: orjson<4,>=3.9.15
-Requires-Dist: mlrun-pipelines-kfp-common~=0.3.15
-Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.3.10; python_version < "3.11"
+Requires-Dist: mlrun-pipelines-kfp-common~=0.4.4
+Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.4.3
 Requires-Dist: docstring_parser~=0.16
 Requires-Dist: aiosmtplib~=3.0
 Provides-Extra: s3
@@ -102,7 +103,7 @@ Requires-Dist: taos-ws-py==0.3.2; extra == "tdengine"
 Provides-Extra: snowflake
 Requires-Dist: snowflake-connector-python~=3.7; extra == "snowflake"
 Provides-Extra: kfp18
-Requires-Dist: mlrun_pipelines_kfp_v1_8[kfp]>=0.3.2; python_version < "3.11" and extra == "kfp18"
+Requires-Dist: mlrun_pipelines_kfp_v1_8[kfp]>=0.4.0; python_version < "3.11" and extra == "kfp18"
 Provides-Extra: api
 Requires-Dist: uvicorn~=0.32.1; extra == "api"
 Requires-Dist: dask-kubernetes~=0.11.0; extra == "api"
@@ -118,7 +119,7 @@ Requires-Dist: timelength~=1.1; extra == "api"
 Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "api"
 Requires-Dist: aiosmtplib~=3.0; extra == "api"
 Requires-Dist: pydantic<2,>=1; extra == "api"
-Requires-Dist: mlrun-pipelines-kfp-v1-8[kfp]~=0.3.10; python_version < "3.11" and extra == "api"
+Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.4.3; extra == "api"
 Requires-Dist: grpcio~=1.70.0; extra == "api"
 Provides-Extra: all
 Requires-Dist: adlfs==2023.9.0; extra == "all"
@@ -212,7 +213,7 @@ Requires-Dist: igz-mgmt~=0.4.1; extra == "complete-api"
 Requires-Dist: kafka-python~=2.1.0; extra == "complete-api"
 Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "complete-api"
 Requires-Dist: mlflow~=2.16; extra == "complete-api"
-Requires-Dist: mlrun-pipelines-kfp-v1-8[kfp]~=0.3.10; python_version < "3.11" and extra == "complete-api"
+Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.4.3; extra == "complete-api"
 Requires-Dist: msrest~=0.6.21; extra == "complete-api"
 Requires-Dist: objgraph~=3.6; extra == "complete-api"
 Requires-Dist: oss2==2.18.1; extra == "complete-api"

{mlrun-1.8.0rc62.dist-info → mlrun-1.9.0.dist-info}/RECORD

@@ -1,6 +1,6 @@
 mlrun/__init__.py,sha256=Cqm9U9eCEdLpMejhU2BEhubu0mHL71igJJIwYa738EA,7450
-mlrun/__main__.py,sha256=0NDzPf9VFRO8KFfGgb8mkGUPIDS285aASV8Hbxs-ND0,45920
-mlrun/config.py,sha256=xxmIe0g1YP2Y5_R_uQjP3w6v_4lA7iTjXhAyojdY08I,71929
+mlrun/__main__.py,sha256=ktsoMxLmW65ABu5oG7TbUJvusyAYy_PRollQOGj4cGY,46352
+mlrun/config.py,sha256=R8RDtYt2EZ4lU-H_oi8cSx52Zw33C92MBe9Lv4JqHl8,71933
 mlrun/errors.py,sha256=LkcbXTLANGdsgo2CRX2pdbyNmt--lMsjGv0XZMgP-Nc,8222
 mlrun/execution.py,sha256=rss4zA5M9tOCnSaXrK_-_BQ5F5DfF9OzesgQliq7jvQ,50008
 mlrun/features.py,sha256=ReBaNGsBYXqcbgI012n-SO_j6oHIbk_Vpv0CGPXbUmo,15842
@@ -21,7 +21,7 @@ mlrun/artifacts/manager.py,sha256=bqp2-VgThx5RAGEui6LwTA9EMNNq6Vu1Z_-yjBpk92c,16
 mlrun/artifacts/model.py,sha256=J5b8zODrpx5ULtsgS9RGKqzMXYs7ADacE0BLBglmhrs,22239
 mlrun/artifacts/plots.py,sha256=TxOHBaGbj7fEKNTHVIM_uxQjqPLpU3Rh1pqGh2_inuo,4833
 mlrun/common/__init__.py,sha256=xY3wHC4TEJgez7qtnn1pQvHosi8-5UJOCtyGBS7FcGE,571
-mlrun/common/constants.py,sha256=14xMUX9C5BB-LxsTlMTJQf_Xz2DyRjaK9yeR5dadcDU,3426
+mlrun/common/constants.py,sha256=Yj5YHANbpKHDKxZ1y5bV1wifvV3UQZ2QuvECUuYhrpM,3594
 mlrun/common/helpers.py,sha256=DIdqs_eN3gO5bZ8iFobIvx8cEiOxYxhFIyut6-O69T0,1385
 mlrun/common/secrets.py,sha256=vc8WV82EZsCB5ENjUkObFOzZP59aZ1w8F82PTnqwBnc,5181
 mlrun/common/types.py,sha256=1gxThbmC0Vd0U1ffIkEwz4T4S7JOgHt70rvw8TCO21c,1073
@@ -86,7 +86,7 @@ mlrun/datastore/alibaba_oss.py,sha256=k-OHVe08HjMewlkpsT657CbOiVFAfSq9_EqhCE-k86
 mlrun/datastore/azure_blob.py,sha256=SzAcHYSXkm8Zpopz2Ea-rWVClH0URocUazcNK04S9W0,12776
 mlrun/datastore/base.py,sha256=9R3lwB_L4hv5WW2q24WS62_KTh-wO4UG6pwzISZU6bM,26231
 mlrun/datastore/datastore.py,sha256=AXXPgHpSG8Ig1RtTDGfdCJu4UT-AQPC43FGBOptIVOg,9484
-mlrun/datastore/datastore_profile.py,sha256=JIH7GYlC40pdaqO8xwYp52dO8QFEB8le021kFINFh4k,23862
+mlrun/datastore/datastore_profile.py,sha256=ywIFYRzMWgb510GuRBDhsJvUbkPENl2qUDQbIBifuWE,23865
 mlrun/datastore/dbfs_store.py,sha256=QkDRzwFnvm7CgEg4NuGxes6tBgKDyhX0CiBUvK8c9pk,6568
 mlrun/datastore/filestore.py,sha256=OcykjzhbUAZ6_Cb9bGAXRL2ngsOpxXSb4rR0lyogZtM,3773
 mlrun/datastore/google_cloud_storage.py,sha256=MnToY6irdhBZ8Wcapqnr1Yq2724LAh2uPO7MAtdWfUY,8716
@@ -110,7 +110,7 @@ mlrun/db/__init__.py,sha256=WqJ4x8lqJ7ZoKbhEyFqkYADd9P6E3citckx9e9ZLcIU,1163
 mlrun/db/auth_utils.py,sha256=hpg8D2r82oN0BWabuWN04BTNZ7jYMAF242YSUpK7LFM,5211
 mlrun/db/base.py,sha256=lfPEPUBXPdmzXUhFD0hDBBWXdV7HXfcsz9Gj_AMLllg,30819
 mlrun/db/factory.py,sha256=yP2vVmveUE7LYTCHbS6lQIxP9rW--zdISWuPd_I3d_4,2111
-mlrun/db/httpdb.py,sha256=p2IIJxkc_q3xAKymIzX3Ic3ep60X6bpQ6PZRe9s0fyY,232763
+mlrun/db/httpdb.py,sha256=AxfZuJQJl8yGv9rt2pPOPMrDik3bZ73gbdyEfeEZvyM,232833
 mlrun/db/nopdb.py,sha256=ttC1pe95rZdMgiLG9kzrjZFYB1gWj3SEqeqK5c0q0w4,27197
 mlrun/feature_store/__init__.py,sha256=SlI845bWt6xX34SXunHHqhmFAR9-5v2ak8N-qpcAPGo,1328
 mlrun/feature_store/api.py,sha256=qKj5Tk6prTab6XWatWhBuPRVp0eJEctoxRMN2wz48vA,32168
@@ -198,13 +198,13 @@ mlrun/frameworks/sklearn/metrics_library.py,sha256=PVPNpljaGglXcw7THLHX-XU0cA8Nu
 mlrun/frameworks/sklearn/mlrun_interface.py,sha256=JzHMBQM4sPBJqzb8P-rsG_2RQ_QrXrb8KFj6GNfwrJ0,14339
 mlrun/frameworks/sklearn/model_handler.py,sha256=n0vpsQznva_WVloz7GTnfMGcMDQU_f1bHhUAJ_qxjfE,4753
 mlrun/frameworks/sklearn/utils.py,sha256=Cg_pSxUMvKe8vBSLQor6JM8u9_ccKJg4Rk5EPDzTsVo,1209
-mlrun/frameworks/tf_keras/__init__.py,sha256=OIHqn6Y1_lcB2j5O3hKea4cp0uVtF4Lv96lvAkm_9x4,10553
+mlrun/frameworks/tf_keras/__init__.py,sha256=M2sMbYHLrlF-KFR5kvA9mevRo3Nf8U0B5a_DM9rzwCY,10484
 mlrun/frameworks/tf_keras/mlrun_interface.py,sha256=1xPUv8YAqxrY3CmkMfWMdp2yEAvk5viiMH6qw41ytSk,16617
-mlrun/frameworks/tf_keras/model_handler.py,sha256=K99NF7WxqmGpzXXssO9mdQb_Swaek_g3aIhZ4GW-k1k,28276
+mlrun/frameworks/tf_keras/model_handler.py,sha256=CquUawDowzZgbQtphFbdz1QBzNlOfnnePSHkHDS9Iqg,31102
 mlrun/frameworks/tf_keras/model_server.py,sha256=PZW6OBGTJ6bSfHedAhhW8HATbJyp2VaAzSDC02zjyKk,9653
-mlrun/frameworks/tf_keras/utils.py,sha256=_QWk1YmdRybbUB54vsQFE2_WMuAK0g7eR1ozVbMk0Go,4284
+mlrun/frameworks/tf_keras/utils.py,sha256=Z8hA1CgpSJWLC_T6Ay7xZKVyWlX9B85MSmQr2biXRag,4582
 mlrun/frameworks/tf_keras/callbacks/__init__.py,sha256=sd8aWG2jO9mO_noZca0ReVf8X6fSCqO_di1Z-mT8FH8,742
-mlrun/frameworks/tf_keras/callbacks/logging_callback.py,sha256=QGgCDAVtoixy94NdrbCfQLsV0kQlv4b535pVv47RRRw,22029
+mlrun/frameworks/tf_keras/callbacks/logging_callback.py,sha256=0-cwQU-GIPUnHRqez9MX4_Gwgb6OgdcPRjU6E2Xo5JE,22323
 mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py,sha256=O63ZL_04wqxDHhDsNhGX880dUmIUPt5bKnSgTUTbduw,8838
 mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py,sha256=Z0Qgea9-G0-9gq3OPH3Cre7f8zMWX0bWyBQPgxxemYQ,28616
 mlrun/frameworks/xgboost/__init__.py,sha256=NyFRxu5v5z8oegbJP05PFUmfJL3I3JeN1PYHjIbfXpo,10335
@@ -213,7 +213,7 @@ mlrun/frameworks/xgboost/model_handler.py,sha256=e3VLKMmaC9OFoclUPx9buUXYLDe1Ab3
 mlrun/frameworks/xgboost/utils.py,sha256=5zLzHoeI3n2FuA_rdGzi404QCTLfQx1TYEyUWhZogs8,1069
 mlrun/launcher/__init__.py,sha256=JL8qkT1lLr1YvW6iP0hmwDTaSR2RfrMDx0-1gWRhTOE,571
 mlrun/launcher/base.py,sha256=uZaUpwjy9_Z137aQ4b1JsuYqD01ZVRxytAxZSFKSu6U,16480
-mlrun/launcher/client.py,sha256=lJ3y9brmPspgwAZrUPAeu3Dn5B7mh9k3MhrbFKhNDvw,6286
+mlrun/launcher/client.py,sha256=iZS5rqIf2do1XNGJ4oyQHkdQtndr48l9Mffs_xCI_NI,6280
 mlrun/launcher/factory.py,sha256=RW7mfzEFi8fR0M-4W1JQg1iq3_muUU6OTqT_3l4Ubrk,2338
 mlrun/launcher/local.py,sha256=775HY-8S9LFUX5ubGXrLO0N1lVh8bn-DHFmNYuNqQPA,11451
 mlrun/launcher/remote.py,sha256=rLJW4UAnUT5iUb4BsGBOAV3K4R29a0X4lFtRkVKlyYU,7709
@@ -242,8 +242,8 @@ mlrun/model_monitoring/db/tsdb/helpers.py,sha256=0oUXc4aUkYtP2SGP6jTb3uPPKImIUsV
 mlrun/model_monitoring/db/tsdb/tdengine/__init__.py,sha256=vgBdsKaXUURKqIf3M0y4sRatmSVA4CQiJs7J5dcVBkQ,620
 mlrun/model_monitoring/db/tsdb/tdengine/schemas.py,sha256=EslhaR65jfeNdD5Ibk-3Hb4e5r5qYPfHb9rTChX3sG0,12689
 mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py,sha256=Uadj0UvAmln2MxDWod-kAzau1uNlqZh981rPhbUH_5M,2857
-mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connection.py,sha256=8xo2O_yQrJGNDoYYB3Bwtdwwvzs3U9dT3BtPot0zENQ,6449
-mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=h0ZrNgOwTlBRd_DaYDc6eeVM9f_8CLJMUPEAIrZpbyU,37803
+mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connection.py,sha256=dtkaHaWKWERPXylEWMECeetwrz3rWl0P43AADcTjlls,9330
+mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=f-3bycfwGvu7gX_Jh3vRT-s2k_yZRXLeIkBrt11xTdc,37998
 mlrun/model_monitoring/db/tsdb/v3io/__init__.py,sha256=aL3bfmQsUQ-sbvKGdNihFj8gLCK3mSys0qDcXtYOwgc,616
 mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py,sha256=_-zo9relCDtjGgievxAcAP9gVN9nDWs8BzGtFwTjb9M,6284
 mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=aDdyHWJLOG-3EgPTJgbEzOOQf5NetpFQk_HdQbs5_zI,47160
@@ -271,9 +271,9 @@ mlrun/platforms/iguazio.py,sha256=6VBTq8eQ3mzT96tzjYhAtcMQ2VjF4x8LpIPW5DAcX2Q,13
 mlrun/projects/__init__.py,sha256=0Krf0WIKfnZa71WthYOg0SoaTodGg3sV_hK3f_OlTPI,1220
 mlrun/projects/operations.py,sha256=9ntpM8WnnyRk1iCY0NDKW-3aR4j2QZPSJM0SdMnsQKs,20032
 mlrun/projects/pipelines.py,sha256=wud7ezeEmhIJvfYE_wzQbA4ygEfGXHtbOtoOpan6poY,48556
-mlrun/projects/project.py,sha256=O3oeg5ZZLvsvftN_T6syB1xQJ7e1eFg37mrlIaL4VCY,237075
+mlrun/projects/project.py,sha256=VY9Wd1GfMH8CUUdOPN9UDsmcvsddlB_HYd2abodvSXY,237350
 mlrun/runtimes/__init__.py,sha256=J9Sy2HiyMlztNv6VUurMzF5H2XzttNil8nRsWDsqLyg,8923
-mlrun/runtimes/base.py,sha256=m347abcZRdI8654wBJea9D2GBRlkv_prz9UrA5ihlkA,37859
+mlrun/runtimes/base.py,sha256=K-38p2HYSpaPcenJgEzQqLlLVFxfwLu2Bwe9oXGWCTM,37853
 mlrun/runtimes/daskjob.py,sha256=JwuGvOiPsxEDHHMMUS4Oie4hLlYYIZwihAl6DjroTY0,19521
 mlrun/runtimes/funcdoc.py,sha256=zRFHrJsV8rhDLJwoUhcfZ7Cs0j-tQ76DxwUqdXV_Wyc,9810
 mlrun/runtimes/function_reference.py,sha256=fnMKUEieKgy4JyVLhFpDtr6JvKgOaQP8F_K2H3-Pk9U,5030
@@ -283,7 +283,7 @@ mlrun/runtimes/local.py,sha256=yedo3R1c46cB1mX7aOz8zORXswQPvX86U-_fYxXoqTY,22717
 mlrun/runtimes/mounts.py,sha256=2dkoktm3TXHe4XHmRhvC0UfvWzq2vy_13MeaW7wgyPo,18735
 mlrun/runtimes/pod.py,sha256=kjnDKOQKqfmprzA3tbXhaB58Dp6So4cOApcjYZ3kVko,67691
 mlrun/runtimes/remotesparkjob.py,sha256=dod99nqz3GdRfmnBoQKfwFCXTetfuCScd2pKH3HJyoY,7394
-mlrun/runtimes/utils.py,sha256=ut0IvQr47NI4Tw7XeHrA7HU7Zun3B1-K44GInecSWF8,15676
+mlrun/runtimes/utils.py,sha256=VFKA7dWuILAcJGia_7Pw_zBBG00wZlat7o2N6u5EItw,16284
 mlrun/runtimes/databricks_job/__init__.py,sha256=kXGBqhLN0rlAx0kTXhozGzFsIdSqW0uTSKMmsLgq_is,569
 mlrun/runtimes/databricks_job/databricks_cancel_task.py,sha256=sIqIg5DQAf4j0wCPA-G0GoxY6vacRddxCy5KDUZszek,2245
 mlrun/runtimes/databricks_job/databricks_runtime.py,sha256=WBq8Q0RIYwLkyshU7btYrB59wotzK_6xixHqZ-oz0PY,12851
@@ -310,7 +310,7 @@ mlrun/serving/serving_wrapper.py,sha256=R670-S6PX_d5ER6jiHtRvacuPyFzQH0mEf2K0sBI
 mlrun/serving/states.py,sha256=PaVdgZ8GyE8bxr_-RvuSUINwHuPypJmqaQs27aEPweo,73367
 mlrun/serving/utils.py,sha256=k2EIYDWHUGkE-IBI6T0UNT32fw-KySsccIJM_LObI00,4171
 mlrun/serving/v1_serving.py,sha256=c6J_MtpE-Tqu00-6r4eJOCO6rUasHDal9W2eBIcrl50,11853
-mlrun/serving/v2_serving.py,sha256=b3C5Utv2_AOPrH_hPi3NarjNbAK3kRoeIfqMU4qNuUo,25362
+mlrun/serving/v2_serving.py,sha256=kVagsCxjRKzdoj-0PNFOg5iI4ub1EYpvlZkULvo-8pw,25380
 mlrun/track/__init__.py,sha256=yVXbT52fXvGKRlc_ByHqIVt7-9L3DRE634RSeQwgXtU,665
 mlrun/track/tracker.py,sha256=CyTU6Qd3_5GGEJ_hpocOj71wvV65EuFYUjaYEUKAL6Q,3575
 mlrun/track/tracker_manager.py,sha256=IYBl99I62IC6VCCmG1yt6JoHNOQXa53C4DURJ2sWgio,5726
@@ -322,7 +322,7 @@ mlrun/utils/azure_vault.py,sha256=IEFizrDGDbAaoWwDr1WoA88S_EZ0T--vjYtY-i0cvYQ,34
 mlrun/utils/clones.py,sha256=yXOeuLtgIiKZdmjeKK0Z_vIrH19ds5JuoJaCeDjhwOo,7516
 mlrun/utils/condition_evaluator.py,sha256=-nGfRmZzivn01rHTroiGY4rqEv8T1irMyhzxEei-sKc,1897
 mlrun/utils/db.py,sha256=blQgkWMfFH9lcN4sgJQcPQgEETz2Dl_zwbVA0SslpFg,2186
-mlrun/utils/helpers.py,sha256=m5WwqAjneDiF0k9YKBwcwqdE-z0oWd-_4mZvYlNJa18,78461
+mlrun/utils/helpers.py,sha256=ED6uhiximgL9_StrXWnfDNn5uAQFDGIHFOmvQddK6KE,78868
 mlrun/utils/http.py,sha256=t6FrXQstZm9xVVjxqIGiLzrwZNCR4CSienSOuVgNIcI,8706
 mlrun/utils/logger.py,sha256=RG0m1rx6gfkJ-2C1r_p41MMpPiaDYqaYM2lYHDlNZEU,14767
 mlrun/utils/regex.py,sha256=jbR7IiOp6OO0mg9Fl_cVZCpWb9fL9nTPONCUxCDNWXg,5201
@@ -341,11 +341,11 @@ mlrun/utils/notifications/notification/mail.py,sha256=ZyJ3eqd8simxffQmXzqd3bgbAq
 mlrun/utils/notifications/notification/slack.py,sha256=kfhogR5keR7Zjh0VCjJNK3NR5_yXT7Cv-x9GdOUW4Z8,7294
 mlrun/utils/notifications/notification/webhook.py,sha256=zxh8CAlbPnTazsk6r05X5TKwqUZVOH5KBU2fJbzQlG4,5330
 mlrun/utils/version/__init__.py,sha256=7kkrB7hEZ3cLXoWj1kPoDwo4MaswsI2JVOBpbKgPAgc,614
-mlrun/utils/version/version.json,sha256=pftUvz4mfka5ERnvT8qp-923vew4A898akorrF6GIK0,89
+mlrun/utils/version/version.json,sha256=EJ4rtjC3_V0pmJXOcoc2LI3XUF5zyqW2FLmw_PjLKlc,84
 mlrun/utils/version/version.py,sha256=eEW0tqIAkU9Xifxv8Z9_qsYnNhn3YH7NRAfM-pPLt1g,1878
-mlrun-1.8.0rc62.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mlrun-1.8.0rc62.dist-info/METADATA,sha256=_qTx4TQ65fT8WIQ-NuWfELnelYX5HsR-12CEXfcizaU,25805
-mlrun-1.8.0rc62.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-mlrun-1.8.0rc62.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
-mlrun-1.8.0rc62.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.8.0rc62.dist-info/RECORD,,
+mlrun-1.9.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mlrun-1.9.0.dist-info/METADATA,sha256=ulIL0BTmyTEMCClGlH9WA4ZO5-riMYqnCM7FFzVL-Mo,25756
+mlrun-1.9.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mlrun-1.9.0.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
+mlrun-1.9.0.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.9.0.dist-info/RECORD,,

{mlrun-1.8.0rc62.dist-info → mlrun-1.9.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.8.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any