mlrun 1.10.0rc40__py3-none-any.whl → 1.11.0rc16__py3-none-any.whl

mlrun/hub/module.py

@@ -12,76 +12,62 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-import subprocess
-import sys
+import warnings
 from pathlib import Path
 from typing import Optional, Union
 
 import yaml
+from deprecated import deprecated
 
 import mlrun.common.types
 import mlrun.utils
 from mlrun.common.schemas.hub import HubModuleType, HubSourceType
-from mlrun.run import function_to_module, get_object
-from mlrun.utils import logger
+from mlrun.run import get_object
 
-from ..errors import MLRunBadRequestError
-from ..model import ModelObj
 from ..utils import extend_hub_uri_if_needed
+from .base import HubAsset
 
 
-class HubModule(ModelObj):
+class HubModule(HubAsset):
+    ASSET_TYPE = HubSourceType.modules
+
     def __init__(
         self,
         name: str,
+        version: str,
         kind: Union[HubModuleType, str],
-        version: Optional[str] = None,
         description: Optional[str] = None,
         categories: Optional[list] = None,
         requirements: Optional[list] = None,
-        local_path: Optional[str] = None,
+        local_path: Optional[Path] = None,
         filename: Optional[str] = None,
         example: Optional[str] = None,
         url: Optional[str] = None,
         **kwargs,  # catch all for unused args
     ):
-        self.name: str = name
-        self.version: str = version
-        self.kind: HubModuleType = kind
-        self.description: str = description or ""
-        self.categories: list = categories or []
-        self.requirements: list = requirements or []
-        self.local_path: str = local_path or ""
-        self.filename: str = filename or name + ".py"
-        self.example: str = example or ""
-        self.url: str = url or ""
-
-    def module(self):
-        """Import the module after downloading its fils to local_path"""
-        try:
-            return function_to_module(code=self.filename, workdir=self.local_path)
-        except FileNotFoundError:
-            searched_path = self.local_path or "./"
-            raise FileNotFoundError(
-                f"Module file {self.filename} not found in {searched_path}, try calling download_module_files() first"
-            )
-
-    def install_requirements(self) -> None:
-        """
-        Install pip-style requirements (e.g., ["pandas>=2.0.0", "requests==2.31.0"]).
-        """
-        for req in self.requirements:
-            logger.info(f"Installing {req} ...")
-            try:
-                subprocess.run(
-                    [sys.executable, "-m", "pip", "install", req], check=True, text=True
-                )
-                logger.info(f"Installed {req}")
-            except subprocess.CalledProcessError as e:
-                logger.error(f"Failed to install {req} (exit code {e.returncode})")
-
-    def download_module_files(self, local_path=None, secrets=None):
+        super().__init__(
+            name=name,
+            version=version,
+            description=description,
+            categories=categories,
+            requirements=requirements,
+            local_path=local_path,
+            filename=filename,
+            example=example,
+            url=url,
+        )
+        self.kind = kind
+
+    # TODO: Remove this in 1.13.0
+    @deprecated(
+        version="1.11.0",
+        reason="This function is deprecated and will be removed in 1.13. You can download module files by calling "
+        "download_files() instead.",
+        category=FutureWarning,
+    )
+    def download_module_files(
+        self, local_path: Optional[str] = None, secrets: Optional[dict] = None
+    ):
         """
         Download this hub module’s files (code file and, if available, an example notebook) to the target directory
         specified by `local_path` (defaults to the current working directory).
@@ -89,52 +75,50 @@ class HubModule(ModelObj):
         """
         self.local_path = self.verify_directory(path=local_path)
         source_url, _ = extend_hub_uri_if_needed(
-            uri=self.url, asset_type=HubSourceType.modules, file=self.filename
+            uri=self.url, asset_type=self.ASSET_TYPE, file=self.filename
         )
         self._download_object(
             obj_url=source_url, target_name=self.filename, secrets=secrets
         )
         if self.example:
             example_url, _ = extend_hub_uri_if_needed(
-                uri=self.url, asset_type=HubSourceType.modules, file=self.example
+                uri=self.url, asset_type=self.ASSET_TYPE, file=self.example
             )
             self._download_object(
                 obj_url=example_url, target_name=self.example, secrets=secrets
             )
 
-    def _download_object(self, obj_url, target_name, secrets=None):
-        data = get_object(url=obj_url, secrets=secrets)
-        target_dir = self.local_path if self.local_path is not None else os.getcwd()
-        target_filepath = os.path.join(target_dir, target_name)
-        with open(target_filepath, "wb") as f:
-            f.write(data)
-
-    @staticmethod
-    def verify_directory(path: Optional[str] = None) -> Path:
+    def download_files(
+        self,
+        local_path: Optional[str] = None,
+        download_example: bool = True,
+    ):
         """
-        Validate that the given path is an existing directory.
-        If no path has been provided, returns current working directory.
+        Download this hub module’s code file.
+        :param local_path: Target directory to download the module files to. Defaults to the current working directory.
+                           This path will be used to locate the code file when importing it as a module.
+        :param download_example: Whether to download the example notebook if available. Defaults to True.
         """
-        if path:
-            path = Path(path)
-            if not path.exists():
-                raise ValueError(f"Path does not exist: {path}")
-            if not path.is_dir():
-                raise ValueError(f"Path is not a directory: {path}")
-            return path
-        return Path(os.getcwd())
+        super().download_files(
+            local_path=local_path,
+            download_example=download_example,
+        )
 
+    # TODO: Remove this in 1.13.0
+    @deprecated(
+        version="1.11.0",
+        reason="This function is deprecated and will be removed in 1.13. You can get the module source file path by"
+        " calling get_src_file_path() instead.",
+        category=FutureWarning,
+    )
     def get_module_file_path(self):
-        if not self.local_path:
-            raise MLRunBadRequestError(
-                "module files haven't been downloaded yet, try calling download_module_files() first"
-            )
-        return str(Path(self.local_path) / self.filename)
+        """Get the full path to the module's code file."""
+        return super().get_src_file_path()
 
 
 def get_hub_module(
-    url: str = "",
-    download_files: Optional[bool] = True,
+    url: str,
+    download_files: bool = True,
     secrets: Optional[dict] = None,
     local_path: Optional[str] = None,
 ) -> HubModule:
@@ -158,11 +142,18 @@ def get_hub_module(
     spec = item_yaml.pop("spec", {})
     hub_module = HubModule(**item_yaml, **spec, url=url)
     if download_files:
-        hub_module.download_module_files(local_path=local_path, secrets=secrets)
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", category=FutureWarning)
+            hub_module.download_module_files(local_path=local_path, secrets=secrets)
     return hub_module
 
 
-def import_module(url="", install_requirements=False, secrets=None, local_path=None):
+def import_module(
+    url: str,
+    install_requirements: bool = False,
+    secrets: Optional[dict] = None,
+    local_path: Optional[str] = None,
+):
     """
     Import a module from the hub to use directly.
     :param url: hub module url in the format "hub://[<source>/]<item-name>[:<tag>]"

mlrun/hub/step.py

@@ -0,0 +1,113 @@
+# Copyright 2025 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+from typing import Optional
+
+import yaml
+
+from mlrun.common.schemas.hub import HubSourceType
+from mlrun.run import get_object
+
+from ..errors import MLRunInvalidArgumentError
+from ..utils import extend_hub_uri_if_needed
+from .base import HubAsset
+
+
+class HubStep(HubAsset):
+    ASSET_TYPE = HubSourceType.steps
+
+    def __init__(
+        self,
+        name: str,
+        version: str,
+        class_name: str,
+        default_handler: str,
+        description: Optional[str] = None,
+        categories: Optional[list] = None,
+        requirements: Optional[list] = None,
+        local_path: Optional[Path] = None,
+        filename: Optional[str] = None,
+        example: Optional[str] = None,
+        url: Optional[str] = None,
+        **kwargs,  # catch all for unused args
+    ):
+        super().__init__(
+            name=name,
+            version=version,
+            description=description,
+            categories=categories,
+            requirements=requirements,
+            local_path=local_path,
+            filename=filename,
+            example=example,
+            url=url,
+        )
+        self.class_name = class_name
+        self.default_handler = default_handler
+
+    def download_files(
+        self,
+        local_path: Optional[str] = None,
+        download_example: bool = False,
+    ):
+        """
+        Download this step's code file.
+        :param local_path: Target directory to download the step files to. Defaults to the current working directory.
+                           This path will be used to locate the code file when importing it as a python module.
+        :param download_example: Whether to download the example notebook if available. Defaults to False.
+        """
+        super().download_files(
+            local_path=local_path,
+            download_example=download_example,
+        )
+
+
+def get_hub_step(
+    url: str,
+    local_path: Optional[str] = None,
+    download_files: bool = True,
+    include_example: bool = False,
+) -> HubStep:
+    """
+    Get a hub-step object containing metadata of the requested step.
+    :param url: Hub step url in the format "hub://[<source>/]<item-name>[:<tag>]"
+    :param local_path: Path to target directory for the step files. Ignored when download_files is set to False.
+                       Defaults to the current working directory.
+    :param download_files: When set to True, the step code files are downloaded
+    :param include_example: When set to True, the example notebook will also be downloaded (ignored if download_files is
+                           False)
+
+    :return: HubStep object
+    """
+    item_yaml_url, is_hub_uri = extend_hub_uri_if_needed(
+        uri=url, asset_type=HubSourceType.steps, file="item.yaml"
+    )
+    if not is_hub_uri:
+        raise MLRunInvalidArgumentError("Not a valid hub URL")
+    yaml_obj = get_object(url=item_yaml_url)
+    item_yaml = yaml.safe_load(yaml_obj)
+    spec = item_yaml.pop("spec", {})
+    class_name = item_yaml.pop("className", "")
+    default_handler = item_yaml.pop("defaultHandler", "")
+    hub_step = HubStep(
+        **item_yaml,
+        **spec,
+        class_name=class_name,
+        default_handler=default_handler,
+        url=url,
+    )
+    if download_files:
+        hub_step.download_files(local_path=local_path, download_example=include_example)
+    return hub_step

mlrun/launcher/base.py

@@ -16,7 +16,8 @@ import ast
 import copy
 import os
 import uuid
-from typing import Any, Callable, Optional, Union
+from collections.abc import Callable
+from typing import Any, Optional, Union
 
 import mlrun.common.constants
 import mlrun.common.runtimes.constants

mlrun/launcher/local.py

@@ -13,8 +13,9 @@
 # limitations under the License.
 import os
 import pathlib
+from collections.abc import Callable
 from os import environ
-from typing import Callable, Optional, Union
+from typing import Optional, Union
 
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas.schedule

mlrun/model.py

@@ -29,6 +29,7 @@ import pydantic.v1.error_wrappers
 import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas.notification
+import mlrun.common.secrets
 import mlrun.utils.regex
 
 from .utils import (
@@ -235,7 +236,9 @@ class ModelObj:
             fields = list(inspect.signature(cls.__init__).parameters.keys())
 
         if init_with_params:
-            kwargs = {field: struct.pop(field, None) for field in fields}
+            kwargs = {
+                field: struct.pop(field, None) for field in fields if field in struct
+            }
             kwargs.pop("self", None)
             new_obj = cls(**kwargs)
         else:
@@ -1007,6 +1010,7 @@ class RunSpec(ModelObj):
         tolerations=None,
         affinity=None,
         retry=None,
+        auth=None,
     ):
         # A dictionary of parsing configurations that will be read from the inputs the user set. The keys are the inputs
         # keys (parameter names) and the values are the type hint given in the input keys after the colon.
@@ -1048,6 +1052,7 @@ class RunSpec(ModelObj):
         self.tolerations = tolerations or {}
         self.affinity = affinity or {}
         self.retry = retry or {}
+        self.auth = auth or {}
 
     def _serialize_field(
         self, struct: dict, field_name: Optional[str] = None, strip: bool = False
@@ -1616,7 +1621,12 @@ class RunTemplate(ModelObj):
 
         :returns: The RunTemplate object
         """
-
+        if kind == "azure_vault" and isinstance(source, dict):
+            candidate_secret_name = (source.get("k8s_secret") or "").strip()
+            if candidate_secret_name:
+                mlrun.common.secrets.validate_not_forbidden_secret(
+                    candidate_secret_name
+                )
         if kind == "vault" and isinstance(source, list):
             source = {"project": self.metadata.project, "secrets": source}
 

mlrun/model_monitoring/__init__.py

@@ -15,5 +15,4 @@
 from mlrun.common.schemas import ModelEndpoint, ModelEndpointList
 
 from .db import get_tsdb_connector
-from .db._schedules import delete_model_monitoring_schedules_user_folder
 from .helpers import get_stream_path

mlrun/model_monitoring/api.py

@@ -467,7 +467,7 @@ def read_dataset_as_dataframe(
         # Get the features and parse to DataFrame:
         dataset = dataset.get_offline_features(drop_columns=drop_columns).to_dataframe()
 
-    elif isinstance(dataset, (list, np.ndarray)):
+    elif isinstance(dataset, list | np.ndarray):
         if not feature_columns:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "Feature columns list must be provided when dataset input as from type list or numpy array"
@@ -509,7 +509,7 @@ def read_dataset_as_dataframe(
     # Turn the `label_columns` into a list by default:
     if label_columns is None:
         label_columns = []
-    elif isinstance(label_columns, (str, int)):
+    elif isinstance(label_columns, str | int):
         label_columns = [label_columns]
 
     return dataset, label_columns

mlrun/model_monitoring/applications/base.py

@@ -18,7 +18,7 @@ from abc import ABC, abstractmethod
 from collections import defaultdict
 from collections.abc import Iterator
 from contextlib import contextmanager, nullcontext
-from datetime import datetime, timedelta, timezone
+from datetime import UTC, datetime, timedelta
 from typing import Any, Literal, Optional, Union, cast
 
 import pandas as pd
@@ -440,7 +440,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
     ) -> list[tuple[str, str]]:
         if isinstance(endpoints, list):
             if all(
-                isinstance(endpoint, (tuple, list)) and len(endpoint) == 2
+                isinstance(endpoint, tuple | list) and len(endpoint) == 2
                 for endpoint in endpoints
             ):
                 # A list of [(name, uid), ...] / [[name, uid], ...] tuples/lists
@@ -690,8 +690,8 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
 
         # If `start_dt` and `end_dt` do not include time zone information - change them to UTC
         if (start_dt.tzinfo is None) and (end_dt.tzinfo is None):
-            start_dt = start_dt.replace(tzinfo=timezone.utc)
-            end_dt = end_dt.replace(tzinfo=timezone.utc)
+            start_dt = start_dt.replace(tzinfo=UTC)
+            end_dt = end_dt.replace(tzinfo=UTC)
         elif (start_dt.tzinfo is None) or (end_dt.tzinfo is None):
             raise mlrun.errors.MLRunValueError(
                 "The start and end times must either both include time zone information or both be naive (no time "
@@ -850,6 +850,11 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         * ``base_period``, ``int``
         * ``write_output``, ``bool``
         * ``existing_data_handling``, ``str``
+        * ``_init_args``, ``dict`` - the arguments for the application class constructor
+          (equivalent to ``class_arguments``)
+
+        See :py:meth:`~ModelMonitoringApplicationBase.evaluate` for more details
+        about these inputs and params.
 
         For Git sources, add the source archive to the returned job and change the handler:
 
@@ -928,6 +933,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         image: Optional[str] = None,
         with_repo: Optional[bool] = False,
         class_handler: Optional[str] = None,
+        class_arguments: Optional[dict[str, Any]] = None,
         requirements: Optional[Union[str, list[str]]] = None,
         requirements_file: str = "",
         endpoints: Union[list[tuple[str, str]], list[str], Literal["all"], None] = None,
@@ -963,7 +969,10 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                                   You do not need to have a model endpoint to use this option.
         :param image:             Docker image to run the job on (when running remotely).
         :param with_repo:         Whether to clone the current repo to the build source.
-        :param class_handler:     The relative path to the class, useful when using Git sources or code from images.
+        :param class_handler:     The relative path to the application class, useful when using Git sources or code
+                                  from images.
+        :param class_arguments:   The arguments for the application class constructor. These are passed to the
+                                  class ``__init__``. The values must be JSON-serializable.
         :param requirements:      List of Python requirements to be installed in the image.
         :param requirements_file: Path to a Python requirements file to be installed in the image.
         :param endpoints:         The model endpoints to get the data from. The options are:
@@ -1041,7 +1050,9 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
             project=project,
         )
 
-        params: dict[str, Union[list, str, int, None, ds_profile.DatastoreProfile]] = {}
+        params: dict[
+            str, Union[list, dict, str, int, None, ds_profile.DatastoreProfile]
+        ] = {}
         if endpoints:
             params["endpoints"] = endpoints
             if sample_data is None:
@@ -1077,6 +1088,9 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                 )
         params["stream_profile"] = stream_profile
 
+        if class_arguments:
+            params["_init_args"] = class_arguments
+
         inputs: dict[str, str] = {}
         for data, identifier in [
             (sample_data, "sample_data"),

mlrun/model_monitoring/applications/context.py

@@ -249,6 +249,7 @@ class MonitoringApplicationContext:
                 project=self.project_name,
                 endpoint_id=self.endpoint_id,
                 feature_analysis=True,
+                tsdb_metrics=False,
             )
         return self._model_endpoint
 

mlrun/model_monitoring/controller.py

@@ -139,12 +139,10 @@ class _BatchWindow:
             self._start, self._stop - self._step + 1, self._step
         ):
             entered = True
-            start_time = datetime.datetime.fromtimestamp(
-                timestamp, tz=datetime.timezone.utc
-            )
+            start_time = datetime.datetime.fromtimestamp(timestamp, tz=datetime.UTC)
             end_time = datetime.datetime.fromtimestamp(
                 timestamp - self.TIMESTAMP_RESOLUTION_MICRO + self._step,
-                tz=datetime.timezone.utc,
+                tz=datetime.UTC,
             )
             yield _Interval(start_time, end_time)
 
@@ -164,23 +162,15 @@ class _BatchWindow:
                     # If the last analyzed time is earlier than the stop time,
                     # yield the final partial interval from last_analyzed to stop
                     yield _Interval(
-                        datetime.datetime.fromtimestamp(
-                            last_analyzed, tz=datetime.timezone.utc
-                        ),
-                        datetime.datetime.fromtimestamp(
-                            self._stop, tz=datetime.timezone.utc
-                        ),
+                        datetime.datetime.fromtimestamp(last_analyzed, tz=datetime.UTC),
+                        datetime.datetime.fromtimestamp(self._stop, tz=datetime.UTC),
                     )
             else:
                 # The time span between the start and end of the batch is shorter than the step,
                 # so we need to yield a partial interval covering that range.
                 yield _Interval(
-                    datetime.datetime.fromtimestamp(
-                        self._start, tz=datetime.timezone.utc
-                    ),
-                    datetime.datetime.fromtimestamp(
-                        self._stop, tz=datetime.timezone.utc
-                    ),
+                    datetime.datetime.fromtimestamp(self._start, tz=datetime.UTC),
+                    datetime.datetime.fromtimestamp(self._stop, tz=datetime.UTC),
                 )
 
             self._update_last_analyzed(last_analyzed=self._stop)
@@ -866,7 +856,7 @@ class MonitoringApplicationController:
                     last_request = last_request_dict.get(endpoint.metadata.uid, None)
                     if isinstance(last_request, float):
                         last_request = datetime.datetime.fromtimestamp(
-                            last_request, tz=datetime.timezone.utc
+                            last_request, tz=datetime.UTC
                         )
                     elif isinstance(last_request, pd.Timestamp):
                         last_request = last_request.to_pydatetime()

mlrun/model_monitoring/db/_schedules.py

@@ -13,15 +13,12 @@
 # limitations under the License.
 
 import json
-import sys
 from abc import ABC, abstractmethod
 from contextlib import AbstractContextManager
 from datetime import datetime
 from types import TracebackType
 from typing import TYPE_CHECKING, Final, Optional
 
-import botocore.exceptions
-
 import mlrun
 import mlrun.common.schemas as schemas
 import mlrun.errors
@@ -30,10 +27,7 @@ import mlrun.utils.helpers
 from mlrun.utils import logger
 
 if TYPE_CHECKING:
-    if sys.version_info >= (3, 11):
-        from typing import Self
-    else:
-        from typing_extensions import Self
+    from typing import Self
 
 
 class ModelMonitoringSchedulesFileBase(AbstractContextManager, ABC):
@@ -88,16 +82,8 @@ class ModelMonitoringSchedulesFileBase(AbstractContextManager, ABC):
         except (
             mlrun.errors.MLRunNotFoundError,
             # Different errors are raised for S3 or local storage, see ML-8042
-            botocore.exceptions.ClientError,
             FileNotFoundError,
-        ) as err:
-            if (
-                isinstance(err, botocore.exceptions.ClientError)
-                # Add a log only to "NoSuchKey" errors codes - equivalent to `FileNotFoundError`
-                and err.response["Error"]["Code"] != "NoSuchKey"
-            ):
-                raise
-
+        ):
             logger.exception(
                 "The schedules file was not found. It should have been created "
                 "as a part of the model endpoint's creation",

mlrun/model_monitoring/db/_stats.py

@@ -15,10 +15,9 @@ import abc
 import json
 import typing
 from abc import abstractmethod
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from typing import cast
 
-import botocore.exceptions
 import fsspec
 
 import mlrun.datastore.base
@@ -83,23 +82,13 @@ class ModelMonitoringStatsFile(abc.ABC):
             content = json.loads(self._item.get().decode())
             timestamp = content.get("timestamp")
             if timestamp is not None:
-                timestamp = datetime.fromisoformat(timestamp).astimezone(
-                    tz=timezone.utc
-                )
+                timestamp = datetime.fromisoformat(timestamp).astimezone(tz=UTC)
             return content.get("data"), timestamp
         except (
             mlrun.errors.MLRunNotFoundError,
             # Different errors are raised for S3 or local storage, see ML-8042
-            botocore.exceptions.ClientError,
             FileNotFoundError,
         ) as err:
-            if (
-                isinstance(err, botocore.exceptions.ClientError)
-                # Add a log only to "NoSuchKey" errors codes - equivalent to `FileNotFoundError`
-                and err.response["Error"]["Code"] != "NoSuchKey"
-            ):
-                raise
-
             logger.warning(
                 "The Stats file was not found. It should have been created "
                 "as a part of the model endpoint's creation",