mlrun 1.10.0rc18__py3-none-any.whl → 1.11.0rc16__py3-none-any.whl

mlrun/launcher/local.py

@@ -13,8 +13,9 @@
 # limitations under the License.
 import os
 import pathlib
+from collections.abc import Callable
 from os import environ
-from typing import Callable, Optional, Union
+from typing import Optional, Union
 
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas.schedule
@@ -243,6 +244,8 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
 
         # if the handler has module prefix force "local" (vs "handler") runtime
         kind = "local" if isinstance(handler, str) and "." in handler else ""
+
+        # Create temporary local function for execution
         fn = mlrun.new_function(meta.name, command=command, args=args, kind=kind)
         fn.metadata = meta
         setattr(fn, "_is_run_local", True)

mlrun/model.py

@@ -29,6 +29,7 @@ import pydantic.v1.error_wrappers
 import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas.notification
+import mlrun.common.secrets
 import mlrun.utils.regex
 
 from .utils import (
@@ -235,7 +236,9 @@ class ModelObj:
             fields = list(inspect.signature(cls.__init__).parameters.keys())
 
         if init_with_params:
-            kwargs = {field: struct.pop(field, None) for field in fields}
+            kwargs = {
+                field: struct.pop(field, None) for field in fields if field in struct
+            }
             kwargs.pop("self", None)
             new_obj = cls(**kwargs)
         else:
@@ -667,7 +670,7 @@ class ImageBuilder(ModelObj):
         """
         requirements = requirements or []
         self._verify_list(requirements, "requirements")
-        resolved_requirements = self._resolve_requirements(
+        resolved_requirements = self.resolve_requirements(
             requirements, requirements_file
         )
         requirements = self.requirements or [] if not overwrite else []
@@ -680,7 +683,7 @@ class ImageBuilder(ModelObj):
         self.requirements = requirements
 
     @staticmethod
-    def _resolve_requirements(requirements: list, requirements_file: str = "") -> list:
+    def resolve_requirements(requirements: list, requirements_file: str = "") -> list:
         requirements = requirements or []
         requirements_to_resolve = []
 
@@ -1007,6 +1010,7 @@ class RunSpec(ModelObj):
         tolerations=None,
         affinity=None,
         retry=None,
+        auth=None,
     ):
         # A dictionary of parsing configurations that will be read from the inputs the user set. The keys are the inputs
         # keys (parameter names) and the values are the type hint given in the input keys after the colon.
@@ -1048,6 +1052,7 @@ class RunSpec(ModelObj):
         self.tolerations = tolerations or {}
         self.affinity = affinity or {}
         self.retry = retry or {}
+        self.auth = auth or {}
 
     def _serialize_field(
         self, struct: dict, field_name: Optional[str] = None, strip: bool = False
@@ -1616,7 +1621,12 @@ class RunTemplate(ModelObj):
 
         :returns: The RunTemplate object
         """
-
+        if kind == "azure_vault" and isinstance(source, dict):
+            candidate_secret_name = (source.get("k8s_secret") or "").strip()
+            if candidate_secret_name:
+                mlrun.common.secrets.validate_not_forbidden_secret(
+                    candidate_secret_name
+                )
         if kind == "vault" and isinstance(source, list):
             source = {"project": self.metadata.project, "secrets": source}
 

mlrun/model_monitoring/__init__.py

@@ -15,5 +15,4 @@
 from mlrun.common.schemas import ModelEndpoint, ModelEndpointList
 
 from .db import get_tsdb_connector
-from .db._schedules import delete_model_monitoring_schedules_user_folder
 from .helpers import get_stream_path

mlrun/model_monitoring/api.py

@@ -18,6 +18,7 @@ from datetime import datetime
 
 import numpy as np
 import pandas as pd
+from deprecated import deprecated
 
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.datastore.base
@@ -29,8 +30,9 @@ from mlrun.common.schemas.model_monitoring import (
     FunctionURI,
 )
 from mlrun.data_types.infer import InferOptions, get_df_stats
-from mlrun.utils import datetime_now, logger
+from mlrun.utils import check_if_hub_uri, datetime_now, logger, merge_requirements
 
+from ..common.schemas.hub import HubModuleType
 from .helpers import update_model_endpoint_last_request
 
 # A union of all supported dataset types:
@@ -45,6 +47,14 @@ DatasetType = typing.Union[
 ]
 
 
+# TODO: Remove this in 1.12.0
+@deprecated(
+    version="1.10.0",
+    reason="This function is deprecated and will be removed in 1.12. You can generate a model endpoint by either "
+    "deploying a monitored serving function as a real-time service or running it as an offline job. "
+    "To retrieve model endpoints, use `project.list_model_endpoints()`",
+    category=FutureWarning,
+)
 def get_or_create_model_endpoint(
     project: str,
     model_endpoint_name: str,
@@ -67,8 +77,8 @@ def get_or_create_model_endpoint(
     :param model_endpoint_name:      If a new model endpoint is created, the model endpoint name will be presented
                                      under this endpoint (applicable only to new endpoint_id).
     :param model_path:               The model store path (applicable only to new endpoint_id).
-    :param endpoint_id:              Model endpoint unique ID. If not exist in DB, will generate a new record based
-                                     on the provided `endpoint_id`.
+    :param endpoint_id:              Model endpoint unique ID. If not exist in DB, will generate a new record with a
+                                     newly generated ID.
     :param function_name:            If a new model endpoint is created, use this function name.
     :param function_tag:             If a new model endpoint is created, use this function tag.
     :param context:                  MLRun context. If `function_name` not provided, use the context to generate the
@@ -91,25 +101,26 @@ def get_or_create_model_endpoint(
         function_name = FunctionURI.from_string(
             context.to_dict()["spec"]["function"]
         ).function
-    try:
-        model_endpoint = db_session.get_model_endpoint(
-            project=project,
-            name=model_endpoint_name,
-            endpoint_id=endpoint_id,
-            function_name=function_name,
-            function_tag=function_tag or "latest",
-            feature_analysis=feature_analysis,
-        )
-        # If other fields provided, validate that they are correspond to the existing model endpoint data
-        _model_endpoint_validations(
-            model_endpoint=model_endpoint,
-            model_path=model_path,
-            sample_set_statistics=sample_set_statistics,
-        )
+    if endpoint_id or function_name:
+        try:
+            model_endpoint = db_session.get_model_endpoint(
+                project=project,
+                name=model_endpoint_name,
+                endpoint_id=endpoint_id,
+                function_name=function_name,
+                function_tag=function_tag or "latest",
+                feature_analysis=feature_analysis,
+            )
+            # If other fields provided, validate that they are correspond to the existing model endpoint data
+            _model_endpoint_validations(
+                model_endpoint=model_endpoint,
+                model_path=model_path,
+                sample_set_statistics=sample_set_statistics,
+            )
 
-    except (mlrun.errors.MLRunNotFoundError, mlrun.errors.MLRunInvalidArgumentError):
-        # Create a new model endpoint with the provided details
-        pass
+        except mlrun.errors.MLRunNotFoundError:
+            # Create a new model endpoint with the provided details
+            pass
     if not model_endpoint:
         model_endpoint = _generate_model_endpoint(
             project=project,
@@ -123,6 +134,13 @@ def get_or_create_model_endpoint(
     return model_endpoint
 
 
+# TODO: Remove this in 1.12.0
+@deprecated(
+    version="1.10.0",
+    reason="This function is deprecated and will be removed in 1.12. "
+    "Instead, run a monitored serving function as a job",
+    category=FutureWarning,
+)
 def record_results(
     project: str,
     model_path: str,
@@ -144,8 +162,8 @@ def record_results(
     :param model_path:               The model Store path.
     :param model_endpoint_name:      If a new model endpoint is generated, the model endpoint name will be presented
                                      under this endpoint.
-    :param endpoint_id:              Model endpoint unique ID. If not exist in DB, will generate a new record based
-                                     on the provided `endpoint_id`.
+    :param endpoint_id:              Model endpoint unique ID. If not exist in DB, will generate a new record with a
+                                     newly generated ID.
     :param function_name:            If a new model endpoint is created, use this function name for generating the
                                      function URI.
     :param context:                  MLRun context. Note that the context is required generating the model endpoint.
@@ -236,6 +254,7 @@ def _model_endpoint_validations(
             key=model_obj.key,
             iter=model_obj.iter,
             tree=model_obj.tree,
+            uid=model_obj.uid,
         )
 
         # Enrich the uri schema with the store prefix
@@ -325,12 +344,15 @@ def _generate_model_endpoint(
 
     :return `mlrun.common.schemas.ModelEndpoint` object.
     """
+
     current_time = datetime_now()
     model_endpoint = mlrun.common.schemas.ModelEndpoint(
         metadata=mlrun.common.schemas.ModelEndpointMetadata(
             project=project,
             name=model_endpoint_name,
             endpoint_type=mlrun.common.schemas.model_monitoring.EndpointType.BATCH_EP,
+            # Due to backwards compatibility, this endpoint will be created as a legacy batch endpoint.
+            mode=mlrun.common.schemas.model_monitoring.EndpointMode.BATCH_LEGACY,
         ),
         spec=mlrun.common.schemas.ModelEndpointSpec(
             function_name=function_name or "function",
@@ -445,7 +467,7 @@ def read_dataset_as_dataframe(
         # Get the features and parse to DataFrame:
         dataset = dataset.get_offline_features(drop_columns=drop_columns).to_dataframe()
 
-    elif isinstance(dataset, (list, np.ndarray)):
+    elif isinstance(dataset, list | np.ndarray):
         if not feature_columns:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "Feature columns list must be provided when dataset input as from type list or numpy array"
@@ -487,7 +509,7 @@ def read_dataset_as_dataframe(
     # Turn the `label_columns` into a list by default:
     if label_columns is None:
         label_columns = []
-    elif isinstance(label_columns, (str, int)):
+    elif isinstance(label_columns, str | int):
         label_columns = [label_columns]
 
     return dataset, label_columns
@@ -527,8 +549,9 @@ def _create_model_monitoring_function_base(
     name: typing.Optional[str] = None,
     image: typing.Optional[str] = None,
     tag: typing.Optional[str] = None,
-    requirements: typing.Union[str, list[str], None] = None,
+    requirements: typing.Union[list[str], None] = None,
     requirements_file: str = "",
+    local_path: typing.Optional[str] = None,
     **application_kwargs,
 ) -> mlrun.runtimes.ServingRuntime:
     """
@@ -540,12 +563,26 @@ def _create_model_monitoring_function_base(
             "An application cannot have the following names: "
             f"{mm_constants._RESERVED_FUNCTION_NAMES}"
         )
-    if name and name.endswith(mm_constants._RESERVED_EVALUATE_FUNCTION_SUFFIX):
+    _, has_valid_suffix, suffix = mlrun.utils.helpers.ensure_batch_job_suffix(name)
+    if name and not has_valid_suffix:
         raise mlrun.errors.MLRunValueError(
-            "Model monitoring application names cannot end with `-batch`"
+            f"Model monitoring application names cannot end with `{suffix}`"
         )
     if func is None:
         func = ""
+    if check_if_hub_uri(func):
+        hub_module = mlrun.get_hub_module(url=func, local_path=local_path)
+        if hub_module.kind != HubModuleType.monitoring_app:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "The provided module is not a monitoring application"
+            )
+        requirements = mlrun.model.ImageBuilder.resolve_requirements(
+            requirements, requirements_file
+        )
+        requirements = merge_requirements(
+            reqs_priority=requirements, reqs_secondary=hub_module.requirements
+        )
+        func = hub_module.get_module_file_path()
     func_obj = typing.cast(
         mlrun.runtimes.ServingRuntime,
         mlrun.code_to_function(

mlrun/model_monitoring/applications/__init__.py

@@ -12,6 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .base import ModelMonitoringApplicationBase
+from .base import ExistingDataHandling, ModelMonitoringApplicationBase
 from .context import MonitoringApplicationContext
 from .results import ModelMonitoringApplicationMetric, ModelMonitoringApplicationResult