mlrun 1.10.0rc40__py3-none-any.whl → 1.11.0rc16__py3-none-any.whl

mlrun/errors.py

@@ -264,12 +264,14 @@ class MLRunFatalFailureError(Exception):
 
 
 class ModelRunnerError(MLRunBaseError):
-    def __init__(self, models_errors: dict[str:str], *args) -> None:
+    def __init__(self, models_errors: dict[str, str], *args) -> None:
         self.models_errors = models_errors
         super().__init__(self.__repr__(), *args)
 
     def __repr__(self):
-        return f"ModelRunnerError: {repr(self.models_errors)}"
+        return "ModelRunnerError: " + ";\n".join(
+            f"{model} {msg}" for model, msg in self.models_errors.items()
+        )
 
     def __copy__(self):
         return type(self)(models_errors=self.models_errors)

mlrun/execution.py

@@ -15,7 +15,6 @@
 import logging
 import os
 import uuid
-import warnings
 from copy import deepcopy
 from typing import Optional, Union, cast
 
@@ -101,6 +100,7 @@ class MLClientCtx:
         self._node_selector = {}
         self._tolerations = {}
         self._affinity = {}
+        self._auth = {}
 
         self._function = ""
         self._parameters = {}
@@ -234,6 +234,11 @@ class MLClientCtx:
         """Dictionary with node selectors (read-only)"""
         return deepcopy(self._node_selector)
 
+    @property
+    def auth(self):
+        """Dictionary with auth (read-only)"""
+        return deepcopy(self._auth)
+
     @property
     def tolerations(self):
         """Dictionary with tolerations (read-only)"""
@@ -437,6 +442,7 @@ class MLClientCtx:
             self._affinity = spec.get("affinity", self._affinity)
             self._reset_on_run = spec.get("reset_on_run", self._reset_on_run)
             self._retry_spec = spec.get("retry", self._retry_spec)
+            self._auth = spec.get("auth", self._auth)
 
         self._init_dbs(rundb)
 
@@ -1141,14 +1147,6 @@ class MLClientCtx:
         self._update_run()
         return item
 
-    def get_cached_artifact(self, key):
-        """Return a logged artifact from cache (for potential updates)"""
-        warnings.warn(
-            "get_cached_artifact is deprecated in 1.8.0 and will be removed in 1.11.0. Use get_artifact instead.",
-            FutureWarning,
-        )
-        return self.get_artifact(key)
-
     def get_artifact(
         self, key, tag=None, iter=None, tree=None, uid=None
     ) -> Optional[Artifact]:
@@ -1310,6 +1308,7 @@ class MLClientCtx:
                 "node_selector": self._node_selector,
                 "tolerations": self._tolerations,
                 "affinity": self._affinity,
+                "auth": self._auth,
                 "retry": self._retry_spec,
             },
             "status": {
@@ -1511,15 +1510,15 @@ class MLClientCtx:
 
 
 def _cast_result(value):
-    if isinstance(value, (int, str, float)):
+    if isinstance(value, int | str | float):
         return value
     if isinstance(value, list):
         return [_cast_result(v) for v in value]
     if isinstance(value, dict):
         return {k: _cast_result(v) for k, v in value.items()}
-    if isinstance(value, (np.int64, np.integer)):
+    if isinstance(value, np.int64 | np.integer):
         return int(value)
-    if isinstance(value, (np.floating, np.float64)):
+    if isinstance(value, np.floating | np.float64):
         return float(value)
     if isinstance(value, np.ndarray):
         return value.tolist()

mlrun/feature_store/api.py

@@ -361,7 +361,7 @@ def _ingest(
         import pyspark.sql
 
         if (
-            isinstance(source, (pd.DataFrame, pyspark.sql.DataFrame))
+            isinstance(source, pd.DataFrame | pyspark.sql.DataFrame)
             and run_config is not None
         ):
             raise mlrun.errors.MLRunInvalidArgumentError(

mlrun/feature_store/common.py

@@ -256,7 +256,7 @@ class RunConfig:
     @function.setter
     def function(self, function):
         if function and not (
-            isinstance(function, (str, FunctionReference)) or hasattr(function, "apply")
+            isinstance(function, str | FunctionReference) or hasattr(function, "apply")
         ):
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "function must be a uri (string) or mlrun function object/reference"

mlrun/feature_store/feature_vector_utils.py

@@ -370,7 +370,7 @@ class OnlineVectorService:
         if (
             not entity_rows
             or not isinstance(entity_rows, list)
-            or not isinstance(entity_rows[0], (list, dict))
+            or not isinstance(entity_rows[0], list | dict)
         ):
             raise mlrun.errors.MLRunInvalidArgumentError(
                 f"input data is of type {type(entity_rows)}. must be a list of lists or list of dicts"

mlrun/feature_store/steps.py

@@ -279,11 +279,11 @@ class MapValues(StepToDict, MLRunStep):
                     new_col_type = df.schema[new_column_name].dataType
                     #  in order to avoid exception at isna on non-decimal/float columns -
                     #  we need to check their types before filtering.
-                    if isinstance(col_type, (FloatType, DoubleType, DecimalType)):
+                    if isinstance(col_type, FloatType | DoubleType | DecimalType):
                         column_filter = (~isnull(col(column))) & (~isnan(col(column)))
                     else:
                         column_filter = ~isnull(col(column))
-                    if isinstance(new_col_type, (FloatType, DoubleType, DecimalType)):
+                    if isinstance(new_col_type, FloatType | DoubleType | DecimalType):
                         new_column_filter = isnull(col(new_column_name)) | isnan(
                             col(new_column_name)
                         )
@@ -295,7 +295,7 @@ class MapValues(StepToDict, MLRunStep):
                         for k, v in column_map.items()
                         if v is None
                         or (
-                            isinstance(v, (float, np.float64, np.float32, np.float16))
+                            isinstance(v, float | np.float64 | np.float32 | np.float16)
                             and math.isnan(v)
                         )
                     ]
@@ -338,7 +338,7 @@ class MapValues(StepToDict, MLRunStep):
                     for val in column_map.values()
                     if type(val) is not None
                     and not (
-                        isinstance(val, (float, np.float64, np.float32, np.float16))
+                        isinstance(val, float | np.float64 | np.float32 | np.float16)
                         and math.isnan(val)
                     )
                 )
@@ -358,7 +358,9 @@ class MapValues(StepToDict, MLRunStep):
                         and val != "-inf"
                         and val != "inf"
                         and not (
-                            isinstance(val, (float, np.float64, np.float32, np.float16))
+                            isinstance(
+                                val, float | np.float64 | np.float32 | np.float16
+                            )
                             and math.isnan(val)
                         )
                     )
@@ -443,7 +445,7 @@ class OneHotEncoder(StepToDict, MLRunStep):
         self.mapping = mapping
         for key, values in mapping.items():
             for val in values:
-                if not (isinstance(val, str) or isinstance(val, (int, np.integer))):
+                if not (isinstance(val, str) or isinstance(val, int | np.integer)):
                     raise mlrun.errors.MLRunInvalidArgumentError(
                         "For OneHotEncoder you must provide int or string mapping list"
                     )

mlrun/frameworks/_common/utils.py

@@ -97,9 +97,9 @@ class CommonUtils(ABC):
         """
         if isinstance(dataset, np.ndarray):
             return dataset
-        if isinstance(dataset, (pd.DataFrame, pd.Series)):
+        if isinstance(dataset, pd.DataFrame | pd.Series):
             return dataset.to_numpy()
-        if isinstance(dataset, (list, tuple)):
+        if isinstance(dataset, list | tuple):
             return np.array(dataset)
         if isinstance(dataset, dict):
             return np.array(list(dataset.values()))
@@ -133,7 +133,7 @@ class CommonUtils(ABC):
         """
         if isinstance(dataset, pd.DataFrame):
             return dataset
-        if isinstance(dataset, (np.ndarray, pd.Series, list, tuple, dict)):
+        if isinstance(dataset, np.ndarray | pd.Series | list | tuple | dict):
             return pd.DataFrame(dataset)
         try:
             # SciPy is not in MLRun's requirements but common to all frameworks.

mlrun/frameworks/_dl_common/loggers/logger.py

@@ -281,7 +281,7 @@ class Logger:
         """
         for parameter_name, parameter_value in self._context.parameters.items():
             # Check if the parameter is a trackable value:
-            if isinstance(parameter_value, (str, bool, float, int)):
+            if isinstance(parameter_value, str | bool | float | int):
                 self.log_static_hyperparameter(
                     parameter_name=parameter_name, value=parameter_value
                 )

mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py

@@ -14,8 +14,9 @@
 
 import os
 from abc import abstractmethod
+from collections.abc import Callable
 from datetime import datetime
-from typing import Any, Callable, Generic, Optional, Union
+from typing import Any, Generic, Optional, Union
 
 import yaml
 

mlrun/frameworks/_ml_common/loggers/mlrun_logger.py

@@ -73,7 +73,7 @@ class MLRunLogger(Logger):
         """
         for parameter_name, parameter_value in self._context.parameters.items():
             # Check if the parameter is a trackable value:
-            if isinstance(parameter_value, (str, bool, float, int)):
+            if isinstance(parameter_value, str | bool | float | int):
                 self.log_static_hyperparameter(
                     parameter_name=parameter_name, value=parameter_value
                 )

mlrun/frameworks/_ml_common/utils.py

@@ -13,8 +13,9 @@
 # limitations under the License.
 
 from abc import ABC
+from collections.abc import Callable
 from enum import Enum
-from typing import Callable, Union
+from typing import Union
 
 import pandas as pd
 from sklearn.base import is_classifier, is_regressor

mlrun/frameworks/auto_mlrun/auto_mlrun.py

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Callable, Optional, Union
+from collections.abc import Callable
+from typing import Optional, Union
 
 import mlrun
 from mlrun.artifacts import get_model
@@ -58,7 +59,7 @@ def get_framework_by_instance(model: CommonTypes.ModelType) -> str:
 
         from mlrun.frameworks.xgboost import XGBoostModelHandler
 
-        if isinstance(model, (XGBModel, Booster)):
+        if isinstance(model, XGBModel | Booster):
             return XGBoostModelHandler.FRAMEWORK_NAME
     except ModuleNotFoundError:
         pass
@@ -69,7 +70,7 @@ def get_framework_by_instance(model: CommonTypes.ModelType) -> str:
 
         from mlrun.frameworks.lgbm import LGBMModelHandler
 
-        if isinstance(model, (LGBMModel, Booster)):
+        if isinstance(model, LGBMModel | Booster):
             return LGBMModelHandler.FRAMEWORK_NAME
     except ModuleNotFoundError:
         pass

mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py

@@ -13,8 +13,9 @@
 # limitations under the License.
 
 from abc import ABC
+from collections.abc import Callable
 from types import ModuleType
-from typing import Callable, Optional, Union
+from typing import Optional, Union
 
 import lightgbm as lgb
 

mlrun/frameworks/onnx/dataset.py

@@ -13,7 +13,8 @@
 # limitations under the License.
 
 import math
-from typing import Callable, Optional, Union
+from collections.abc import Callable
+from typing import Optional, Union
 
 import numpy as np
 

mlrun/frameworks/onnx/mlrun_interface.py

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Callable, Optional
+from collections.abc import Callable
+from typing import Optional
 
 import numpy as np
 import onnx

mlrun/frameworks/pytorch/callbacks/logging_callback.py

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Callable, Optional, Union
+from collections.abc import Callable
+from typing import Optional, Union
 
 import numpy as np
 from torch import Tensor
@@ -496,7 +497,7 @@ class LoggingCallback(Callback):
             value = self._objects[source]
             for key in key_chain:
                 try:
-                    if isinstance(value, (dict, list, tuple)):
+                    if isinstance(value, dict | list | tuple):
                         value = value[key]
                     else:
                         value = getattr(value, key)
@@ -507,7 +508,7 @@ class LoggingCallback(Callback):
                     )
 
         # Parse the value:
-        if isinstance(value, (Tensor, Parameter)):
+        if isinstance(value, Tensor | Parameter):
             if value.numel() == 1:
                 value = float(value)
             else:
@@ -523,7 +524,7 @@ class LoggingCallback(Callback):
                     f"The parameter with the following key chain: {key_chain} is a numpy.ndarray with {value.size} "
                     f"elements. numpy arrays are trackable only if they have 1 element."
                 )
-        elif not isinstance(value, (float, int, str, bool)):
+        elif not isinstance(value, float | int | str | bool):
             raise mlrun.errors.MLRunInvalidArgumentError(
                 f"The parameter with the following key chain: {key_chain} is of type '{type(value)}'. "
                 f"The only trackable types are: float, int, str and bool."

mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Callable, Optional, Union
+from collections.abc import Callable
+from typing import Optional, Union
 
 import torch
 from torch import Tensor

mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py

@@ -12,8 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from collections.abc import Callable
 from datetime import datetime
-from typing import Callable, Optional, Union
+from typing import Optional, Union
 
 import torch
 from torch import Tensor

mlrun/frameworks/pytorch/utils.py

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Callable, Union
+from collections.abc import Callable
+from typing import Union
 
 import numpy as np
 import torch

mlrun/frameworks/sklearn/metric.py

@@ -15,7 +15,8 @@
 import importlib
 import json
 import sys
-from typing import Callable, Optional, Union
+from collections.abc import Callable
+from typing import Optional, Union
 
 import mlrun.errors
 

mlrun/frameworks/tf_keras/callbacks/logging_callback.py

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Callable, Optional, Union
+from collections.abc import Callable
+from typing import Optional, Union
 
 import numpy as np
 import tensorflow as tf
@@ -441,8 +442,8 @@ class LoggingCallback(Callback):
                     )
 
         # Parse the value:
-        if isinstance(value, (tf.Tensor, tf.Variable)) or (
-            is_keras_3() and isinstance(value, (keras.KerasTensor, keras.Variable))
+        if isinstance(value, tf.Tensor | tf.Variable) or (
+            is_keras_3() and isinstance(value, keras.KerasTensor | keras.Variable)
         ):
             if int(tf.size(value)) == 1:
                 value = float(value)
@@ -459,7 +460,7 @@ class LoggingCallback(Callback):
                     f"The parameter with the following key chain: {key_chain} is a numpy.ndarray with {value.size} "
                     f"elements. numpy arrays are trackable only if they have 1 element."
                 )
-        elif not (isinstance(value, (float, int, str, bool))):
+        elif not (isinstance(value, float | int | str | bool)):
             raise mlrun.errors.MLRunInvalidArgumentError(
                 f"The parameter with the following key chain: {key_chain} is of type '{type(value)}'. The only "
                 f"trackable types are: float, int, str and bool."

mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Callable, Optional, Union
+from collections.abc import Callable
+from typing import Optional, Union
 
 import mlrun
 from mlrun.artifacts import Artifact

mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py

@@ -12,8 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from collections.abc import Callable
 from datetime import datetime
-from typing import Callable, Optional, Union
+from typing import Optional, Union
 
 import tensorflow as tf
 from packaging import version

mlrun/hub/__init__.py

@@ -11,5 +11,42 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from typing import Optional
+
+import mlrun
+from mlrun.common.schemas.hub import HubSourceType
 
 from .module import get_hub_module, import_module
+from .step import get_hub_step
+
+
+def get_hub_item(
+    source_name: str,
+    item_name: str,
+    version: Optional[str] = None,
+    tag: Optional[str] = "latest",
+    force_refresh: bool = False,
+    item_type: HubSourceType = HubSourceType.functions,
+) -> mlrun.common.schemas.hub.HubItem:
+    """
+    Retrieve a specific hub item.
+
+    :param source_name: Name of source.
+    :param item_name: Name of the item to retrieve, as it appears in the hub catalog.
+    :param version: Get a specific version of the item. Default is ``None``.
+    :param tag: Get a specific version of the item identified by tag. Default is ``latest``.
+    :param force_refresh: Make the server fetch the information from the actual hub
+        source, rather than
+        rely on cached information. Default is ``False``.
+    :param item_type: The type of item to retrieve from the hub source (e.g: functions, modules).
+    :returns: :py:class:`~mlrun.common.schemas.hub.HubItem`.
+    """
+    db = mlrun.get_run_db()
+    return db.get_hub_item(
+        source_name=source_name,
+        item_name=item_name,
+        version=version,
+        tag=tag,
+        force_refresh=force_refresh,
+        item_type=item_type,
+    )

mlrun/hub/base.py

@@ -0,0 +1,142 @@
+# Copyright 2025 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import ClassVar, Optional
+
+from mlrun.common.schemas.hub import HubSourceType
+from mlrun.run import function_to_module, get_object
+from mlrun.utils import logger
+
+from ..errors import MLRunBadRequestError
+from ..model import ModelObj
+from ..utils import extend_hub_uri_if_needed
+
+
+class HubAsset(ModelObj):
+    ASSET_TYPE: ClassVar[HubSourceType]
+
+    def __init__(
+        self,
+        name: str,
+        version: str,
+        description: Optional[str] = None,
+        categories: Optional[list] = None,
+        requirements: Optional[list] = None,
+        local_path: Optional[Path] = None,
+        filename: Optional[str] = None,
+        example: Optional[str] = None,
+        url: Optional[str] = None,
+        **kwargs,
+    ):
+        self.name: str = name
+        self.version: str = version
+        self.description: str = description or ""
+        self.categories: list = categories or []
+        self.requirements: list = requirements or []
+        self.local_path: Optional[Path] = local_path
+        self.filename: str = filename or name
+        self.example: str = example or ""
+        self.url: str = url or ""
+
+    def module(self):
+        """Import the code of the asset as a module."""
+        try:
+            return function_to_module(code=self.filename, workdir=self.local_path)
+        except Exception as e:
+            raise MLRunBadRequestError(
+                f"Failed to import module from {self.get_src_file_path()}: {e}"
+            )
+
+    def install_requirements(self) -> None:
+        """
+        Install pip-style requirements of the asset (e.g., ["pandas>=2.0.0", "requests==2.31.0"]).
+        """
+        if not self.requirements or len(self.requirements) == 0:
+            logger.info("No requirements to install.")
+            return
+        for req in self.requirements:
+            logger.info(f"Installing {req} ...")
+            try:
+                subprocess.run(
+                    [sys.executable, "-m", "pip", "install", req], check=True, text=True
+                )
+                logger.info(f"Installed {req}")
+            except subprocess.CalledProcessError as e:
+                logger.error(f"Failed to install {req} (exit code {e.returncode})")
+
+    def download_files(
+        self,
+        local_path: Optional[str] = None,
+        download_example: bool = True,
+    ):
+        """
+        Download this hub asset’s files (code file and, if available and requested, an example notebook) to the target
+        directory specified by `local_path` (defaults to the current working directory).
+        This path will be used later to locate the code file when calling module().
+        """
+        self.local_path = self.verify_directory(path=local_path)
+        source_url, _ = extend_hub_uri_if_needed(
+            uri=self.url, asset_type=self.ASSET_TYPE, file=self.filename
+        )
+        self._download_object(obj_url=source_url, target_name=self.filename)
+        if download_example and self.example:
+            example_url, _ = extend_hub_uri_if_needed(
+                uri=self.url, asset_type=self.ASSET_TYPE, file=self.example
+            )
+            self._download_object(obj_url=example_url, target_name=self.example)
+
+    def _download_object(self, obj_url, target_name, secrets=None):
+        data = get_object(url=obj_url, secrets=secrets)
+        target_filepath = os.path.join(self.local_path, target_name)
+        with open(target_filepath, "wb") as f:
+            f.write(data)
+
+    @staticmethod
+    def verify_directory(path: Optional[str] = None) -> Path:
+        """
+        Validate that the given path is an existing directory.
+        If no path has been provided, returns current working directory.
+        """
+        if path:
+            path = Path(path)
+            if not path.exists():
+                raise ValueError(f"Path does not exist: {path}")
+            if not path.is_dir():
+                raise ValueError(f"Path is not a directory: {path}")
+            return path
+        return Path(os.getcwd())
+
+    def get_src_file_path(self) -> str:
+        """Get the full path to the asset's code file."""
+        if not self.local_path:
+            raise MLRunBadRequestError(
+                f"Local path not set. Call download_files() first to download the asset files, or "
+                f"set_local_path() with the directory containing {self.filename}"
+            )
+        src_path = Path(self.local_path) / self.filename
+        if not src_path.exists():
+            raise FileNotFoundError(
+                f"File {self.filename} not found in {self.local_path}. Call download_files() first to download the "
+                f"asset files, or set_local_path() with the directory containing {self.filename}"
+            )
+
+        return str(src_path)
+
+    def set_local_path(self, path: str):
+        """Set the local path where the asset's files are stored."""
+        self.local_path = self.verify_directory(path=path)