mlrun 1.10.0rc18__py3-none-any.whl → 1.11.0rc16__py3-none-any.whl

mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py

@@ -12,8 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from collections.abc import Callable
 from datetime import datetime
-from typing import Callable, Optional, Union
+from typing import Optional, Union
 
 import tensorflow as tf
 from packaging import version

mlrun/hub/__init__.py

@@ -0,0 +1,52 @@
+# Copyright 2025 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Optional
+
+import mlrun
+from mlrun.common.schemas.hub import HubSourceType
+
+from .module import get_hub_module, import_module
+from .step import get_hub_step
+
+
+def get_hub_item(
+    source_name: str,
+    item_name: str,
+    version: Optional[str] = None,
+    tag: Optional[str] = "latest",
+    force_refresh: bool = False,
+    item_type: HubSourceType = HubSourceType.functions,
+) -> mlrun.common.schemas.hub.HubItem:
+    """
+    Retrieve a specific hub item.
+
+    :param source_name: Name of source.
+    :param item_name: Name of the item to retrieve, as it appears in the hub catalog.
+    :param version: Get a specific version of the item. Default is ``None``.
+    :param tag: Get a specific version of the item identified by tag. Default is ``latest``.
+    :param force_refresh: Make the server fetch the information from the actual hub
+        source, rather than
+        rely on cached information. Default is ``False``.
+    :param item_type: The type of item to retrieve from the hub source (e.g: functions, modules).
+    :returns: :py:class:`~mlrun.common.schemas.hub.HubItem`.
+    """
+    db = mlrun.get_run_db()
+    return db.get_hub_item(
+        source_name=source_name,
+        item_name=item_name,
+        version=version,
+        tag=tag,
+        force_refresh=force_refresh,
+        item_type=item_type,
+    )

mlrun/hub/base.py

@@ -0,0 +1,142 @@
+# Copyright 2025 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import ClassVar, Optional
+
+from mlrun.common.schemas.hub import HubSourceType
+from mlrun.run import function_to_module, get_object
+from mlrun.utils import logger
+
+from ..errors import MLRunBadRequestError
+from ..model import ModelObj
+from ..utils import extend_hub_uri_if_needed
+
+
+class HubAsset(ModelObj):
+    ASSET_TYPE: ClassVar[HubSourceType]
+
+    def __init__(
+        self,
+        name: str,
+        version: str,
+        description: Optional[str] = None,
+        categories: Optional[list] = None,
+        requirements: Optional[list] = None,
+        local_path: Optional[Path] = None,
+        filename: Optional[str] = None,
+        example: Optional[str] = None,
+        url: Optional[str] = None,
+        **kwargs,
+    ):
+        self.name: str = name
+        self.version: str = version
+        self.description: str = description or ""
+        self.categories: list = categories or []
+        self.requirements: list = requirements or []
+        self.local_path: Optional[Path] = local_path
+        self.filename: str = filename or name
+        self.example: str = example or ""
+        self.url: str = url or ""
+
+    def module(self):
+        """Import the code of the asset as a module."""
+        try:
+            return function_to_module(code=self.filename, workdir=self.local_path)
+        except Exception as e:
+            raise MLRunBadRequestError(
+                f"Failed to import module from {self.get_src_file_path()}: {e}"
+            )
+
+    def install_requirements(self) -> None:
+        """
+        Install pip-style requirements of the asset (e.g., ["pandas>=2.0.0", "requests==2.31.0"]).
+        """
+        if not self.requirements or len(self.requirements) == 0:
+            logger.info("No requirements to install.")
+            return
+        for req in self.requirements:
+            logger.info(f"Installing {req} ...")
+            try:
+                subprocess.run(
+                    [sys.executable, "-m", "pip", "install", req], check=True, text=True
+                )
+                logger.info(f"Installed {req}")
+            except subprocess.CalledProcessError as e:
+                logger.error(f"Failed to install {req} (exit code {e.returncode})")
+
+    def download_files(
+        self,
+        local_path: Optional[str] = None,
+        download_example: bool = True,
+    ):
+        """
+        Download this hub asset’s files (code file and, if available and requested, an example notebook) to the target
+        directory specified by `local_path` (defaults to the current working directory).
+        This path will be used later to locate the code file when calling module().
+        """
+        self.local_path = self.verify_directory(path=local_path)
+        source_url, _ = extend_hub_uri_if_needed(
+            uri=self.url, asset_type=self.ASSET_TYPE, file=self.filename
+        )
+        self._download_object(obj_url=source_url, target_name=self.filename)
+        if download_example and self.example:
+            example_url, _ = extend_hub_uri_if_needed(
+                uri=self.url, asset_type=self.ASSET_TYPE, file=self.example
+            )
+            self._download_object(obj_url=example_url, target_name=self.example)
+
+    def _download_object(self, obj_url, target_name, secrets=None):
+        data = get_object(url=obj_url, secrets=secrets)
+        target_filepath = os.path.join(self.local_path, target_name)
+        with open(target_filepath, "wb") as f:
+            f.write(data)
+
+    @staticmethod
+    def verify_directory(path: Optional[str] = None) -> Path:
+        """
+        Validate that the given path is an existing directory.
+        If no path has been provided, returns current working directory.
+        """
+        if path:
+            path = Path(path)
+            if not path.exists():
+                raise ValueError(f"Path does not exist: {path}")
+            if not path.is_dir():
+                raise ValueError(f"Path is not a directory: {path}")
+            return path
+        return Path(os.getcwd())
+
+    def get_src_file_path(self) -> str:
+        """Get the full path to the asset's code file."""
+        if not self.local_path:
+            raise MLRunBadRequestError(
+                f"Local path not set. Call download_files() first to download the asset files, or "
+                f"set_local_path() with the directory containing {self.filename}"
+            )
+        src_path = Path(self.local_path) / self.filename
+        if not src_path.exists():
+            raise FileNotFoundError(
+                f"File {self.filename} not found in {self.local_path}. Call download_files() first to download the "
+                f"asset files, or set_local_path() with the directory containing {self.filename}"
+            )
+
+        return str(src_path)
+
+    def set_local_path(self, path: str):
+        """Set the local path where the asset's files are stored."""
+        self.local_path = self.verify_directory(path=path)

mlrun/hub/module.py

@@ -0,0 +1,172 @@
+# Copyright 2025 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import warnings
+from pathlib import Path
+from typing import Optional, Union
+
+import yaml
+from deprecated import deprecated
+
+import mlrun.common.types
+import mlrun.utils
+from mlrun.common.schemas.hub import HubModuleType, HubSourceType
+from mlrun.run import get_object
+
+from ..utils import extend_hub_uri_if_needed
+from .base import HubAsset
+
+
+class HubModule(HubAsset):
+    ASSET_TYPE = HubSourceType.modules
+
+    def __init__(
+        self,
+        name: str,
+        version: str,
+        kind: Union[HubModuleType, str],
+        description: Optional[str] = None,
+        categories: Optional[list] = None,
+        requirements: Optional[list] = None,
+        local_path: Optional[Path] = None,
+        filename: Optional[str] = None,
+        example: Optional[str] = None,
+        url: Optional[str] = None,
+        **kwargs,  # catch all for unused args
+    ):
+        super().__init__(
+            name=name,
+            version=version,
+            description=description,
+            categories=categories,
+            requirements=requirements,
+            local_path=local_path,
+            filename=filename,
+            example=example,
+            url=url,
+        )
+        self.kind = kind
+
+    # TODO: Remove this in 1.13.0
+    @deprecated(
+        version="1.11.0",
+        reason="This function is deprecated and will be removed in 1.13. You can download module files by calling "
+        "download_files() instead.",
+        category=FutureWarning,
+    )
+    def download_module_files(
+        self, local_path: Optional[str] = None, secrets: Optional[dict] = None
+    ):
+        """
+        Download this hub module’s files (code file and, if available, an example notebook) to the target directory
+        specified by `local_path` (defaults to the current working directory).
+        This path will be used later to locate the code file when importing the module.
+        """
+        self.local_path = self.verify_directory(path=local_path)
+        source_url, _ = extend_hub_uri_if_needed(
+            uri=self.url, asset_type=self.ASSET_TYPE, file=self.filename
+        )
+        self._download_object(
+            obj_url=source_url, target_name=self.filename, secrets=secrets
+        )
+        if self.example:
+            example_url, _ = extend_hub_uri_if_needed(
+                uri=self.url, asset_type=self.ASSET_TYPE, file=self.example
+            )
+            self._download_object(
+                obj_url=example_url, target_name=self.example, secrets=secrets
+            )
+
+    def download_files(
+        self,
+        local_path: Optional[str] = None,
+        download_example: bool = True,
+    ):
+        """
+        Download this hub module’s code file.
+        :param local_path: Target directory to download the module files to. Defaults to the current working directory.
+                           This path will be used to locate the code file when importing it as a module.
+        :param download_example: Whether to download the example notebook if available. Defaults to True.
+        """
+        super().download_files(
+            local_path=local_path,
+            download_example=download_example,
+        )
+
+    # TODO: Remove this in 1.13.0
+    @deprecated(
+        version="1.11.0",
+        reason="This function is deprecated and will be removed in 1.13. You can get the module source file path by"
+        " calling get_src_file_path() instead.",
+        category=FutureWarning,
+    )
+    def get_module_file_path(self):
+        """Get the full path to the module's code file."""
+        return super().get_src_file_path()
+
+
+def get_hub_module(
+    url: str,
+    download_files: bool = True,
+    secrets: Optional[dict] = None,
+    local_path: Optional[str] = None,
+) -> HubModule:
+    """
+    Get a hub-module object containing metadata of the requested module.
+    :param url: Hub module url in the format "hub://[<source>/]<item-name>[:<tag>]"
+    :param download_files: When set to True, the module files (code file and example notebook) are downloaded
+    :param secrets: Optional, credentials dict for DB or URL (s3, v3io, ...)
+    :param local_path: Path to target directory for the module files. Ignored when download_files is set to False.
+                       Defaults to the current working directory.
+
+    :return: HubModule object
+    """
+    item_yaml_url, is_hub_uri = extend_hub_uri_if_needed(
+        uri=url, asset_type=HubSourceType.modules, file="item.yaml"
+    )
+    if not is_hub_uri:
+        raise mlrun.errors.MLRunInvalidArgumentError("Not a valid hub URL")
+    yaml_obj = get_object(url=item_yaml_url, secrets=secrets)
+    item_yaml = yaml.safe_load(yaml_obj)
+    spec = item_yaml.pop("spec", {})
+    hub_module = HubModule(**item_yaml, **spec, url=url)
+    if download_files:
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", category=FutureWarning)
+            hub_module.download_module_files(local_path=local_path, secrets=secrets)
+    return hub_module
+
+
+def import_module(
+    url: str,
+    install_requirements: bool = False,
+    secrets: Optional[dict] = None,
+    local_path: Optional[str] = None,
+):
+    """
+    Import a module from the hub to use directly.
+    :param url: hub module url in the format "hub://[<source>/]<item-name>[:<tag>]"
+    :param install_requirements: when set to True, the module's requirements are installed.
+    :param secrets: optional, credentials dict for DB or URL (s3, v3io, ...)
+    :param local_path: Path to target directory for the module files (code and example notebook).
+                       Defaults to the current working directory.
+
+    :return: the module
+    """
+    hub_module: HubModule = get_hub_module(
+        url=url, download_files=True, secrets=secrets, local_path=local_path
+    )
+    if install_requirements:
+        hub_module.install_requirements()
+    return hub_module.module()

mlrun/hub/step.py

@@ -0,0 +1,113 @@
+# Copyright 2025 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+from typing import Optional
+
+import yaml
+
+from mlrun.common.schemas.hub import HubSourceType
+from mlrun.run import get_object
+
+from ..errors import MLRunInvalidArgumentError
+from ..utils import extend_hub_uri_if_needed
+from .base import HubAsset
+
+
+class HubStep(HubAsset):
+    ASSET_TYPE = HubSourceType.steps
+
+    def __init__(
+        self,
+        name: str,
+        version: str,
+        class_name: str,
+        default_handler: str,
+        description: Optional[str] = None,
+        categories: Optional[list] = None,
+        requirements: Optional[list] = None,
+        local_path: Optional[Path] = None,
+        filename: Optional[str] = None,
+        example: Optional[str] = None,
+        url: Optional[str] = None,
+        **kwargs,  # catch all for unused args
+    ):
+        super().__init__(
+            name=name,
+            version=version,
+            description=description,
+            categories=categories,
+            requirements=requirements,
+            local_path=local_path,
+            filename=filename,
+            example=example,
+            url=url,
+        )
+        self.class_name = class_name
+        self.default_handler = default_handler
+
+    def download_files(
+        self,
+        local_path: Optional[str] = None,
+        download_example: bool = False,
+    ):
+        """
+        Download this step's code file.
+        :param local_path: Target directory to download the step files to. Defaults to the current working directory.
+                           This path will be used to locate the code file when importing it as a python module.
+        :param download_example: Whether to download the example notebook if available. Defaults to False.
+        """
+        super().download_files(
+            local_path=local_path,
+            download_example=download_example,
+        )
+
+
+def get_hub_step(
+    url: str,
+    local_path: Optional[str] = None,
+    download_files: bool = True,
+    include_example: bool = False,
+) -> HubStep:
+    """
+    Get a hub-step object containing metadata of the requested step.
+    :param url: Hub step url in the format "hub://[<source>/]<item-name>[:<tag>]"
+    :param local_path: Path to target directory for the step files. Ignored when download_files is set to False.
+                       Defaults to the current working directory.
+    :param download_files: When set to True, the step code files are downloaded
+    :param include_example: When set to True, the example notebook will also be downloaded (ignored if download_files is
+                           False)
+
+    :return: HubStep object
+    """
+    item_yaml_url, is_hub_uri = extend_hub_uri_if_needed(
+        uri=url, asset_type=HubSourceType.steps, file="item.yaml"
+    )
+    if not is_hub_uri:
+        raise MLRunInvalidArgumentError("Not a valid hub URL")
+    yaml_obj = get_object(url=item_yaml_url)
+    item_yaml = yaml.safe_load(yaml_obj)
+    spec = item_yaml.pop("spec", {})
+    class_name = item_yaml.pop("className", "")
+    default_handler = item_yaml.pop("defaultHandler", "")
+    hub_step = HubStep(
+        **item_yaml,
+        **spec,
+        class_name=class_name,
+        default_handler=default_handler,
+        url=url,
+    )
+    if download_files:
+        hub_step.download_files(local_path=local_path, download_example=include_example)
+    return hub_step

mlrun/k8s_utils.py

@@ -26,6 +26,10 @@ from .config import config as mlconfig
 
 _running_inside_kubernetes_cluster = None
 
+K8sObj = typing.Union[kubernetes.client.V1Affinity, kubernetes.client.V1Toleration]
+SanitizedK8sObj = dict[str, typing.Any]
+K8sObjList = typing.Union[list[K8sObj], list[SanitizedK8sObj]]
+
 
 def is_running_inside_kubernetes_cluster():
     global _running_inside_kubernetes_cluster
@@ -232,6 +236,54 @@ def validate_node_selectors(
     return True
 
 
+def sanitize_k8s_objects(
+    k8s_objects: typing.Union[None, K8sObjList, SanitizedK8sObj, K8sObj],
+) -> typing.Union[list[SanitizedK8sObj], SanitizedK8sObj]:
+    """Convert K8s objects to dicts. Handles single objects or lists."""
+    api_client = kubernetes.client.ApiClient()
+    if not k8s_objects:
+        return k8s_objects
+
+    def _sanitize_k8s_object(k8s_obj):
+        return (
+            api_client.sanitize_for_serialization(k8s_obj)
+            if hasattr(k8s_obj, "to_dict")
+            else k8s_obj
+        )
+
+    return (
+        [_sanitize_k8s_object(k8s_obj) for k8s_obj in k8s_objects]
+        if isinstance(k8s_objects, list)
+        else _sanitize_k8s_object(k8s_objects)
+    )
+
+
+def sanitize_scheduling_configuration(
+    tolerations: typing.Optional[list[kubernetes.client.V1Toleration]] = None,
+    affinity: typing.Optional[kubernetes.client.V1Affinity] = None,
+) -> tuple[
+    typing.Optional[list[dict]],
+    typing.Optional[dict],
+]:
+    """
+    Sanitizes pod scheduling configuration for serialization.
+
+    Takes affinity and tolerations and converts them to
+    JSON-serializable dictionaries using the Kubernetes API client's
+    sanitization method.
+
+    Args:
+        affinity: Pod affinity/anti-affinity rules
+        tolerations: List of toleration rules
+
+    Returns:
+        Tuple of (sanitized_affinity, sanitized_tolerations)
+        - affinity: Sanitized dict representation or None
+        - tolerations: List of sanitized dict representations or None
+    """
+    return sanitize_k8s_objects(tolerations), sanitize_k8s_objects(affinity)
+
+
 def enrich_preemption_mode(
     preemption_mode: typing.Optional[str],
     node_selector: dict[str, str],
@@ -269,8 +321,8 @@ def enrich_preemption_mode(
         )
 
     enriched_node_selector = copy.deepcopy(node_selector or {})
-    enriched_tolerations = copy.deepcopy(tolerations or [])
-    enriched_affinity = copy.deepcopy(affinity)
+    enriched_tolerations = _safe_copy_tolerations(tolerations or [])
+    enriched_affinity = _safe_copy_affinity(affinity)
     preemptible_tolerations = generate_preemptible_tolerations()
 
     if handler := _get_mode_handler(preemption_mode):
@@ -288,6 +340,57 @@ def enrich_preemption_mode(
     )
 
 
+def _safe_copy_tolerations(
+    tolerations: list[kubernetes.client.V1Toleration],
+) -> list[kubernetes.client.V1Toleration]:
+    """
+    Safely copy a list of V1Toleration objects without mutating the originals.
+
+    Explicitly reconstructs V1Toleration objects instead of using deepcopy() to avoid
+    serialization errors with K8s client objects that contain threading primitives
+    and non-copyable elements like RLock objects.
+
+    Args:
+        tolerations: List of V1Toleration objects to copy
+
+    Returns:
+        New list containing copied V1Toleration objects with identical field values"""
+    return [
+        kubernetes.client.V1Toleration(
+            effect=toleration.effect,
+            key=toleration.key,
+            value=toleration.value,
+            operator=toleration.operator,
+            toleration_seconds=toleration.toleration_seconds,
+        )
+        for toleration in tolerations
+    ]
+
+
+def _safe_copy_affinity(
+    affinity: kubernetes.client.V1Affinity,
+) -> kubernetes.client.V1Affinity:
+    """
+    Safely create a deep copy of a V1Affinity object.
+
+    Uses K8s API client serialization/deserialization instead of deepcopy() to avoid
+    errors with threading primitives and complex internal structures in K8s objects.
+    Serializes to dict then deserializes back to a clean V1Affinity object.
+
+    Args:
+        affinity: V1Affinity object to copy, or None
+
+    Returns:
+        New V1Affinity object with identical field values, or None if input was None
+    """
+    if not affinity:
+        return None
+    api_client = kubernetes.client.ApiClient()
+    # Convert to dict then back to object properly
+    affinity_dict = api_client.sanitize_for_serialization(affinity)
+    return api_client._ApiClient__deserialize(affinity_dict, "V1Affinity")
+
+
 def _get_mode_handler(mode: str):
     return {
         mlrun.common.schemas.PreemptionModes.prevent: _handle_prevent_mode,
@@ -367,20 +470,6 @@ def _handle_allow_mode(
     list[kubernetes.client.V1Toleration],
     typing.Optional[kubernetes.client.V1Affinity],
 ]:
-    for op in [
-        mlrun.common.schemas.NodeSelectorOperator.node_selector_op_not_in.value,
-        mlrun.common.schemas.NodeSelectorOperator.node_selector_op_in.value,
-    ]:
-        affinity = _prune_affinity_node_selector_requirement(
-            generate_preemptible_node_selector_requirements(op),
-            affinity=affinity,
-        )
-
-    node_selector = _prune_node_selector(
-        mlconfig.get_preemptible_node_selector(),
-        enriched_node_selector=node_selector,
-    )
-
     tolerations = _merge_tolerations(tolerations, preemptible_tolerations)
     return node_selector, tolerations, affinity
 

mlrun/launcher/base.py

@@ -16,7 +16,8 @@ import ast
 import copy
 import os
 import uuid
-from typing import Any, Callable, Optional, Union
+from collections.abc import Callable
+from typing import Any, Optional, Union
 
 import mlrun.common.constants
 import mlrun.common.runtimes.constants
@@ -157,6 +158,19 @@ class BaseLauncher(abc.ABC):
         ]:
             mlrun.utils.helpers.warn_on_deprecated_image(image)
 
+        # Raise an error if retry is configured for a runtime that doesn't support retries.
+        # For local runs, we intentionally skip this validation and allow the run to proceed, since they are typically
+        # used for debugging purposes, and in such cases we avoid blocking their execution.
+        if (
+            not mlrun.runtimes.RuntimeKinds.is_local_runtime(runtime.kind)
+            and run.spec.retry.count
+            and runtime.kind not in mlrun.runtimes.RuntimeKinds.retriable_runtimes()
+        ):
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Retry is not supported for {runtime.kind} runtime, supported runtimes are: "
+                f"{mlrun.runtimes.RuntimeKinds.retriable_runtimes()}"
+            )
+
     @staticmethod
     def _validate_output_path(
         runtime: "mlrun.runtimes.BaseRuntime",
@@ -268,12 +282,6 @@ class BaseLauncher(abc.ABC):
 
         run.metadata.name = mlrun.utils.normalize_name(
             name=name or run.metadata.name or def_name,
-            # if name or runspec.metadata.name are set then it means that is user defined name and we want to warn the
-            # user that the passed name needs to be set without underscore, if its not user defined but rather enriched
-            # from the handler(function) name then we replace the underscore without warning the user.
-            # most of the time handlers will have `_` in the handler name (python convention is to separate function
-            # words with `_`), therefore we don't want to be noisy when normalizing the run name
-            verbose=bool(name or run.metadata.name),
         )
         mlrun.utils.verify_field_regex(
             "run.metadata.name", run.metadata.name, mlrun.utils.regex.run_name