mlrun 1.8.0rc61__py3-none-any.whl → 1.9.0__py3-none-any.whl

mlrun/__main__.py

@@ -13,6 +13,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import functools
+import importlib.metadata
 import json
 import pathlib
 import socket
@@ -25,12 +27,14 @@ from pprint import pprint
 import click
 import dotenv
 import pandas as pd
+import semver
 import yaml
 from tabulate import tabulate
 
 import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
+import mlrun.platforms
 import mlrun.utils.helpers
 from mlrun.common.helpers import parse_versioned_object_uri
 from mlrun.runtimes.mounts import auto_mount as auto_mount_modifier
@@ -63,16 +67,22 @@ from .utils.version import Version
 pd.set_option("mode.chained_assignment", None)
 
 
-def validate_base_argument(ctx, param, value):
+def validate_base_argument(ctx: click.Context, param: click.Parameter, value: str):
+    # click 8.2 expects the context to be passed to make_metavar
+    if semver.VersionInfo.parse(
+        importlib.metadata.version("click")
+    ) < semver.VersionInfo.parse("8.2.0"):
+        metavar_func = functools.partial(param.make_metavar)
+    else:
+        metavar_func = functools.partial(param.make_metavar, ctx)
     if value and value.startswith("-"):
         raise click.BadParameter(
             f"{param.human_readable_name} ({value}) cannot start with '-', ensure the command options are typed "
             f"correctly. Preferably use '--' to separate options and arguments "
-            f"e.g. 'mlrun run --option1 --option2 -- {param.make_metavar()} [--arg1|arg1] [--arg2|arg2]'",
+            f"e.g. 'mlrun run --option1 --option2 -- {metavar_func()} [--arg1|arg1] [--arg2|arg2]'",
             ctx=ctx,
             param=param,
         )
-
     return value
 
 

mlrun/common/constants.py

@@ -90,6 +90,13 @@ class MLRunInternalLabels:
             if not key.startswith("__") and isinstance(value, str)
         ]
 
+    @staticmethod
+    def default_run_labels_to_enrich():
+        return [
+            MLRunInternalLabels.owner,
+            MLRunInternalLabels.v3io_user,
+        ]
+
 
 class DeployStatusTextKind(mlrun.common.types.StrEnum):
     logs = "logs"

mlrun/config.py

@@ -64,7 +64,7 @@ default_config = {
     # url to nuclio dashboard api (can be with user & token, e.g. https://username:password@dashboard-url.com)
     "nuclio_dashboard_url": "",
     "nuclio_version": "",
-    "default_nuclio_runtime": "python:3.9",
+    "default_nuclio_runtime": "python:3.11",
     "nest_asyncio_enabled": "",  # enable import of nest_asyncio for corner cases with old jupyter, set "1"
     "ui_url": "",  # remote/external mlrun UI url (for hyperlinks) (This is deprecated in favor of the ui block)
     "remote_host": "",
@@ -79,7 +79,7 @@ default_config = {
     # comma separated list of images that are in the specified images_registry, and therefore will be enriched with this
     # registry when used. default to mlrun/* which means any image which is of the mlrun repository (mlrun/mlrun,
     # mlrun/ml-base, etc...)
-    "images_to_enrich_registry": "^mlrun/*,python:3.9",
+    "images_to_enrich_registry": "^mlrun/*,^python:3.(9|11)$",
     "kfp_url": "",
     "kfp_ttl": "14400",  # KFP ttl in sec, after that completed PODs will be deleted
     "kfp_image": "mlrun/mlrun-kfp",  # image to use for KFP runner
@@ -286,7 +286,7 @@ default_config = {
             "remote": "mlrun/mlrun",
             "dask": "mlrun/ml-base",
             "mpijob": "mlrun/mlrun",
-            "application": "python:3.9",
+            "application": "python",
         },
         # see enrich_function_preemption_spec for more info,
         # and mlrun.common.schemas.function.PreemptionModes for available options
@@ -482,7 +482,7 @@ default_config = {
             "project_owners_cache_ttl": "30 seconds",
             # access key to be used when the leader is iguazio and polling is done from it
             "iguazio_access_key": "",
-            "iguazio_list_projects_default_page_size": 500,
+            "iguazio_list_projects_default_page_size": 200,
             "iguazio_client_job_cache_ttl": "20 minutes",
             "nuclio_project_deletion_verification_timeout": "300 seconds",
             "nuclio_project_deletion_verification_interval": "5 seconds",

mlrun/datastore/datastore_profile.py

@@ -255,7 +255,7 @@ class DatastoreProfileS3(DatastoreProfile):
     def check_bucket(cls, v):
         if not v:
             warnings.warn(
-                "The 'bucket' attribute will be mandatory starting from version 1.9",
+                "The 'bucket' attribute will be mandatory starting from version 1.10",
                 FutureWarning,
                 stacklevel=2,
             )
@@ -360,7 +360,7 @@ class DatastoreProfileGCS(DatastoreProfile):
     def check_bucket(cls, v):
         if not v:
             warnings.warn(
-                "The 'bucket' attribute will be mandatory starting from version 1.9",
+                "The 'bucket' attribute will be mandatory starting from version 1.10",
                 FutureWarning,
                 stacklevel=2,
             )
@@ -417,7 +417,7 @@ class DatastoreProfileAzureBlob(DatastoreProfile):
     def check_container(cls, v):
         if not v:
             warnings.warn(
-                "The 'container' attribute will be mandatory starting from version 1.9",
+                "The 'container' attribute will be mandatory starting from version 1.10",
                 FutureWarning,
                 stacklevel=2,
             )

mlrun/db/httpdb.py

@@ -21,7 +21,7 @@ import typing
 import warnings
 from copy import deepcopy
 from datetime import datetime, timedelta
-from os import path, remove
+from os import environ, path, remove
 from typing import Literal, Optional, Union
 from urllib.parse import urlparse
 
@@ -129,7 +129,9 @@ class HTTPRunDB(RunDBInterface):
         self._wait_for_background_task_terminal_state_retry_interval = 3
         self._wait_for_project_deletion_interval = 3
         self.client_version = version.Version().get()["version"]
-        self.python_version = str(version.Version().get_python_version())
+        self.python_version = environ.get("MLRUN_PYTHON_VERSION") or str(
+            version.Version().get_python_version()
+        )
 
         self._enrich_and_validate(url)
 

mlrun/frameworks/tf_keras/__init__.py

@@ -14,7 +14,7 @@
 
 from typing import Any, Optional, Union
 
-from tensorflow import keras
+import tensorflow as tf
 
 import mlrun
 import mlrun.common.constants as mlrun_constants
@@ -27,11 +27,11 @@ from .utils import TFKerasTypes, TFKerasUtils
 
 
 def apply_mlrun(
-    model: keras.Model = None,
+    model: tf.keras.Model = None,
     model_name: Optional[str] = None,
     tag: str = "",
     model_path: Optional[str] = None,
-    model_format: str = TFKerasModelHandler.ModelFormats.SAVED_MODEL,
+    model_format: Optional[str] = None,
     save_traces: bool = False,
     modules_map: Optional[Union[dict[str, Union[None, str, list[str]]], str]] = None,
     custom_objects_map: Optional[Union[dict[str, Union[str, list[str]]], str]] = None,
@@ -54,7 +54,7 @@ def apply_mlrun(
     :param model_path:                  The model's store object path. Mandatory for evaluation (to know which model to
                                         update). If model is not provided, it will be loaded from this path.
     :param model_format:                The format to use for saving and loading the model. Should be passed as a
-                                        member of the class 'ModelFormats'. Default: 'ModelFormats.SAVED_MODEL'.
+                                        member of the class 'ModelFormats'.
     :param save_traces:                 Whether or not to use functions saving (only available for the 'SavedModel'
                                         format) for loading the model later without the custom objects dictionary. Only
                                         from tensorflow version >= 2.4.0. Using this setting will increase the model

mlrun/frameworks/tf_keras/callbacks/logging_callback.py

@@ -16,14 +16,14 @@ from typing import Callable, Optional, Union
 
 import numpy as np
 import tensorflow as tf
-from tensorflow import Tensor, Variable
+from tensorflow import keras
 from tensorflow.python.keras.callbacks import Callback
 
 import mlrun
 
 from ..._common import LoggingMode
 from ..._dl_common.loggers import Logger
-from ..utils import TFKerasTypes
+from ..utils import TFKerasTypes, is_keras_3
 
 
 class LoggingCallback(Callback):
@@ -70,7 +70,7 @@ class LoggingCallback(Callback):
                                         {
                                             "epochs": 7
                                         }
-        :param auto_log:                Whether or not to enable auto logging, trying to track common static and dynamic
+        :param auto_log:                Whether to enable auto logging, trying to track common static and dynamic
                                         hyperparameters.
         """
         super().__init__()
@@ -385,18 +385,24 @@ class LoggingCallback(Callback):
             self._logger.log_context_parameters()
 
         # Add learning rate:
-        learning_rate_key = "lr"
-        learning_rate_key_chain = ["optimizer", "lr"]
-        if learning_rate_key not in self._dynamic_hyperparameters_keys and hasattr(
-            self.model, "optimizer"
-        ):
-            try:
-                self._get_hyperparameter(key_chain=learning_rate_key_chain)
+        learning_rate_keys = [
+            "learning_rate",
+            "lr",
+        ]  # "lr" is for backward compatibility in older keras versions.
+        if all(
+            learning_rate_key not in self._dynamic_hyperparameters_keys
+            for learning_rate_key in learning_rate_keys
+        ) and hasattr(self.model, "optimizer"):
+            for learning_rate_key in learning_rate_keys:
+                learning_rate_key_chain = ["optimizer", learning_rate_key]
+                try:
+                    self._get_hyperparameter(key_chain=learning_rate_key_chain)
+                except (KeyError, IndexError, AttributeError, ValueError):
+                    continue
                 self._dynamic_hyperparameters_keys[learning_rate_key] = (
                     learning_rate_key_chain
                 )
-            except (KeyError, IndexError, ValueError):
-                pass
+                break
 
     def _get_hyperparameter(
         self,
@@ -427,7 +433,7 @@ class LoggingCallback(Callback):
                         value = value[key]
                     else:
                         value = getattr(value, key)
-                except KeyError or IndexError as KeyChainError:
+                except KeyError or IndexError or AttributeError as KeyChainError:
                     raise KeyChainError(
                         f"Error during getting a hyperparameter value with the key chain {key_chain}. "
                         f"The {value.__class__} in it does not have the following key/index from the key provided: "
@@ -435,7 +441,9 @@ class LoggingCallback(Callback):
                     )
 
         # Parse the value:
-        if isinstance(value, Tensor) or isinstance(value, Variable):
+        if isinstance(value, (tf.Tensor, tf.Variable)) or (
+            is_keras_3() and isinstance(value, (keras.KerasTensor, keras.Variable))
+        ):
             if int(tf.size(value)) == 1:
                 value = float(value)
             else:
@@ -451,12 +459,7 @@ class LoggingCallback(Callback):
                     f"The parameter with the following key chain: {key_chain} is a numpy.ndarray with {value.size} "
                     f"elements. numpy arrays are trackable only if they have 1 element."
                 )
-        elif not (
-            isinstance(value, float)
-            or isinstance(value, int)
-            or isinstance(value, str)
-            or isinstance(value, bool)
-        ):
+        elif not (isinstance(value, (float, int, str, bool))):
             raise mlrun.errors.MLRunInvalidArgumentError(
                 f"The parameter with the following key chain: {key_chain} is of type '{type(value)}'. The only "
                 f"trackable types are: float, int, str and bool."

mlrun/frameworks/tf_keras/model_handler.py

@@ -29,7 +29,7 @@ from mlrun.features import Feature
 from .._common import without_mlrun_interface
 from .._dl_common import DLModelHandler
 from .mlrun_interface import TFKerasMLRunInterface
-from .utils import TFKerasUtils
+from .utils import TFKerasUtils, is_keras_3
 
 
 class TFKerasModelHandler(DLModelHandler):
@@ -40,8 +40,8 @@ class TFKerasModelHandler(DLModelHandler):
     # Framework name:
     FRAMEWORK_NAME = "tensorflow.keras"
 
-    # Declare a type of an input sample:
-    IOSample = Union[tf.Tensor, tf.TensorSpec, np.ndarray]
+    # Declare a type of input sample (only from keras v3 there is a KerasTensor type):
+    IOSample = Union[tf.Tensor, tf.TensorSpec, "keras.KerasTensor", np.ndarray]
 
     class ModelFormats:
         """
@@ -49,9 +49,19 @@ class TFKerasModelHandler(DLModelHandler):
         """
 
         SAVED_MODEL = "SavedModel"
+        KERAS = "keras"
         H5 = "h5"
         JSON_ARCHITECTURE_H5_WEIGHTS = "json_h5"
 
+        @classmethod
+        def default(cls) -> str:
+            """
+            Get the default model format to use for saving and loading the model based on the keras version.
+
+            :return: The default model format to use.
+            """
+            return cls.KERAS if is_keras_3() else cls.SAVED_MODEL
+
     class _LabelKeys:
         """
         Required labels keys to log with the model.
@@ -65,7 +75,7 @@ class TFKerasModelHandler(DLModelHandler):
         model: keras.Model = None,
         model_path: Optional[str] = None,
         model_name: Optional[str] = None,
-        model_format: str = ModelFormats.SAVED_MODEL,
+        model_format: Optional[str] = None,
         context: mlrun.MLClientCtx = None,
         modules_map: Optional[
             Union[dict[str, Union[None, str, list[str]]], str]
@@ -98,7 +108,7 @@ class TFKerasModelHandler(DLModelHandler):
                                          * If given a loaded model object and the model name is None, the name will be
                                            set to the model's object name / class.
         :param model_format:             The format to use for saving and loading the model. Should be passed as a
-                                         member of the class 'ModelFormats'. Default: 'ModelFormats.SAVED_MODEL'.
+                                         member of the class 'ModelFormats'.
         :param context:                  MLRun context to work with for logging the model.
         :param modules_map:              A dictionary of all the modules required for loading the model. Each key
                                          is a path to a module and its value is the object name to import from it. All
@@ -144,8 +154,11 @@ class TFKerasModelHandler(DLModelHandler):
                                           * 'save_traces' parameter was miss-used.
         """
         # Validate given format:
+        if not model_format:
+            model_format = TFKerasModelHandler.ModelFormats.default()
         if model_format not in [
             TFKerasModelHandler.ModelFormats.SAVED_MODEL,
+            TFKerasModelHandler.ModelFormats.KERAS,
             TFKerasModelHandler.ModelFormats.H5,
             TFKerasModelHandler.ModelFormats.JSON_ARCHITECTURE_H5_WEIGHTS,
         ]:
@@ -153,6 +166,22 @@ class TFKerasModelHandler(DLModelHandler):
                 f"Unrecognized model format: '{model_format}'. Please use one of the class members of "
                 "'TFKerasModelHandler.ModelFormats'"
             )
+        if not is_keras_3():
+            if model_format == TFKerasModelHandler.ModelFormats.KERAS:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "The 'keras' model format is only supported in Keras 3.0.0 and above. "
+                    f"Current version is {keras.__version__}."
+                )
+        else:
+            if (
+                model_format == TFKerasModelHandler.ModelFormats.SAVED_MODEL
+                or model_format
+                == TFKerasModelHandler.ModelFormats.JSON_ARCHITECTURE_H5_WEIGHTS
+            ):
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    f"The '{model_format}' model format is not supported in Keras 3.0.0 and above. "
+                    f"Current version is {keras.__version__}."
+                )
 
         # Validate 'save_traces':
         if save_traces:
@@ -239,11 +268,19 @@ class TFKerasModelHandler(DLModelHandler):
             self._model_file = f"{self._model_name}.h5"
             self._model.save(self._model_file)
 
+        # ModelFormats.keras - Save as a keras file:
+        elif self._model_format == self.ModelFormats.KERAS:
+            self._model_file = f"{self._model_name}.keras"
+            self._model.save(self._model_file)
+
         # ModelFormats.SAVED_MODEL - Save as a SavedModel directory and zip its file:
         elif self._model_format == TFKerasModelHandler.ModelFormats.SAVED_MODEL:
             # Save it in a SavedModel format directory:
+            # Note: Using keras>=3.0.0 can save in this format via `model.export` but then it won't be able to load it
+            # back, only for inference. So, we use the `save` method instead for keras 2 and validate the user won't use
+            # keras 3 and this model format.
             if self._save_traces is True:
-                # Save traces can only be used in versions >= 2.4, so only if its true we use it in the call:
+                # Save traces can only be used in versions >= 2.4, so only if it's true, we use it in the call:
                 self._model.save(self._model_name, save_traces=self._save_traces)
             else:
                 self._model.save(self._model_name)
@@ -303,6 +340,12 @@ class TFKerasModelHandler(DLModelHandler):
                 self._model_file, custom_objects=self._custom_objects
             )
 
+        # ModelFormats.KERAS - Load from a keras file:
+        elif self._model_format == TFKerasModelHandler.ModelFormats.KERAS:
+            self._model = keras.models.load_model(
+                self._model_file, custom_objects=self._custom_objects
+            )
+
         # ModelFormats.SAVED_MODEL - Load from a SavedModel directory:
         elif self._model_format == TFKerasModelHandler.ModelFormats.SAVED_MODEL:
             self._model = keras.models.load_model(
@@ -434,7 +477,10 @@ class TFKerasModelHandler(DLModelHandler):
             )
 
         # Read the inputs:
-        input_signature = [input_layer.type_spec for input_layer in self._model.inputs]
+        input_signature = [
+            getattr(input_layer, "type_spec", input_layer)
+            for input_layer in self._model.inputs
+        ]
 
         # Set the inputs:
         self.set_inputs(from_sample=input_signature)
@@ -453,7 +499,8 @@ class TFKerasModelHandler(DLModelHandler):
 
         # Read the outputs:
         output_signature = [
-            output_layer.type_spec for output_layer in self._model.outputs
+            getattr(output_layer, "type_spec", output_layer)
+            for output_layer in self._model.outputs
         ]
 
         # Set the outputs:
@@ -509,6 +556,17 @@ class TFKerasModelHandler(DLModelHandler):
                     f"'{self._model_path}'"
                 )
 
+        # ModelFormats.KERAS - Get the keras model file:
+        elif self._model_format == TFKerasModelHandler.ModelFormats.KERAS:
+            self._model_file = os.path.join(
+                self._model_path, f"{self._model_name}.keras"
+            )
+            if not os.path.exists(self._model_file):
+                raise mlrun.errors.MLRunNotFoundError(
+                    f"The model file '{self._model_name}.keras' was not found within the given 'model_path': "
+                    f"'{self._model_path}'"
+                )
+
         # ModelFormats.SAVED_MODEL - Get the zip file and extract it, or simply locate the directory:
         elif self._model_format == TFKerasModelHandler.ModelFormats.SAVED_MODEL:
             self._model_file = os.path.join(self._model_path, f"{self._model_name}.zip")
@@ -559,7 +617,9 @@ class TFKerasModelHandler(DLModelHandler):
         # Supported types:
         if isinstance(sample, np.ndarray):
             return super()._read_sample(sample=sample)
-        elif isinstance(sample, tf.TensorSpec):
+        elif isinstance(sample, tf.TensorSpec) or (
+            is_keras_3() and isinstance(sample, keras.KerasTensor)
+        ):
             return Feature(
                 name=sample.name,
                 value_type=TFKerasUtils.convert_tf_dtype_to_value_type(

mlrun/frameworks/tf_keras/utils.py

@@ -11,8 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 import tensorflow as tf
+from packaging import version
 from tensorflow import keras
 
 import mlrun
@@ -117,3 +117,14 @@ class TFKerasUtils(DLUtils):
         raise mlrun.errors.MLRunInvalidArgumentError(
             f"MLRun value type is not supporting the given tensorflow data type: '{tf_dtype}'."
         )
+
+
+def is_keras_3() -> bool:
+    """
+    Check if the current Keras version is 3.x.
+
+    :return: True if Keras version is 3.x, False otherwise.
+    """
+    return hasattr(keras, "__version__") and version.parse(
+        keras.__version__
+    ) >= version.parse("3.0.0")

mlrun/launcher/client.py

@@ -72,7 +72,7 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
     ):
         run.metadata.labels[mlrun_constants.MLRunInternalLabels.kind] = runtime.kind
         mlrun.runtimes.utils.enrich_run_labels(
-            run.metadata.labels, [mlrun.common.runtimes.constants.RunLabels.owner]
+            run.metadata.labels, [mlrun_constants.MLRunInternalLabels.owner]
         )
         if run.spec.output_path:
             run.spec.output_path = run.spec.output_path.replace(