mlrun 1.3.3rc1__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/model.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,13 +16,18 @@ import inspect
 import pathlib
 import re
 import time
+import typing
+import warnings
 from collections import OrderedDict
 from copy import deepcopy
 from datetime import datetime
 from os import environ
 from typing import Any, Dict, List, Optional, Tuple, Union
 
+import pydantic.error_wrappers
+
 import mlrun
+import mlrun.common.schemas.notification
 
 from .utils import (
     dict_to_json,
@@ -57,7 +62,11 @@ class ModelObj:
         return param
 
     def to_dict(self, fields=None, exclude=None):
-        """convert the object to a python dictionary"""
+        """convert the object to a python dictionary
+
+        :param fields:  list of fields to include in the dict
+        :param exclude: list of fields to exclude from the dict
+        """
         struct = {}
         fields = fields or self._dict_fields
         if not fields:
@@ -100,13 +109,19 @@ class ModelObj:
 
         return new_obj
 
-    def to_yaml(self) -> str:
-        """convert the object to yaml"""
-        return dict_to_yaml(self.to_dict())
+    def to_yaml(self, exclude=None) -> str:
+        """convert the object to yaml
 
-    def to_json(self):
-        """convert the object to json"""
-        return dict_to_json(self.to_dict())
+        :param exclude: list of fields to exclude from the yaml
+        """
+        return dict_to_yaml(self.to_dict(exclude=exclude))
+
+    def to_json(self, exclude=None):
+        """convert the object to json
+
+        :param exclude: list of fields to exclude from the json
+        """
+        return dict_to_json(self.to_dict(exclude=exclude))
 
     def to_str(self):
         """convert the object to string (with dict layout)"""
@@ -338,6 +353,7 @@ class ImageBuilder(ModelObj):
         origin_filename=None,
         with_mlrun=None,
         auto_build=None,
+        requirements: list = None,
     ):
         self.functionSourceCode = functionSourceCode  #: functionSourceCode
         self.codeEntryType = ""  #: codeEntryType
@@ -355,6 +371,7 @@ class ImageBuilder(ModelObj):
         self.with_mlrun = with_mlrun  #: with_mlrun
         self.auto_build = auto_build  #: auto_build
         self.build_pod = None
+        self.requirements = requirements or []  #: pip requirements
 
     @property
     def source(self):
@@ -369,12 +386,200 @@ class ImageBuilder(ModelObj):
             or source in [".", "./"]
         ):
             raise mlrun.errors.MLRunInvalidArgumentError(
-                "source must be a compressed (tar.gz / zip) file, a git repo, "
-                "a file path or in the project's context (.)"
+                f"source ({source}) must be a compressed (tar.gz / zip) file, a git repo, "
+                f"a file path or in the project's context (.)"
             )
 
         self._source = source
 
+    def build_config(
+        self,
+        image="",
+        base_image=None,
+        commands: list = None,
+        secret=None,
+        source=None,
+        extra=None,
+        load_source_on_run=None,
+        with_mlrun=None,
+        auto_build=None,
+        requirements=None,
+        requirements_file=None,
+        overwrite=False,
+    ):
+        if image:
+            self.image = image
+        if base_image:
+            self.base_image = base_image
+        if commands:
+            self.with_commands(commands, overwrite=overwrite)
+        if requirements:
+            self.with_requirements(requirements, requirements_file, overwrite=overwrite)
+        if extra:
+            self.extra = extra
+        if secret is not None:
+            self.secret = secret
+        if source:
+            self.source = source
+        if load_source_on_run:
+            self.load_source_on_run = load_source_on_run
+        if with_mlrun is not None:
+            self.with_mlrun = with_mlrun
+        if auto_build:
+            self.auto_build = auto_build
+
+    def with_commands(
+        self,
+        commands: List[str],
+        overwrite: bool = False,
+    ):
+        """add commands to build spec.
+
+        :param commands:  list of commands to run during build
+        :param overwrite: whether to overwrite the existing commands or add to them (the default)
+
+        :return: function object
+        """
+        if not isinstance(commands, list) or not all(
+            isinstance(item, str) for item in commands
+        ):
+            raise ValueError("commands must be a string list")
+        if not self.commands or overwrite:
+            self.commands = commands
+        else:
+            # add commands to existing build commands
+            for command in commands:
+                if command not in self.commands:
+                    self.commands.append(command)
+            # using list(set(x)) won't retain order,
+            # solution inspired from https://stackoverflow.com/a/17016257/8116661
+            self.commands = list(dict.fromkeys(self.commands))
+
+    def with_requirements(
+        self,
+        requirements: Union[str, List[str]],
+        requirements_file: str = "",
+        overwrite: bool = False,
+    ):
+        """add package requirements from file or list to build spec.
+
+        :param requirements:        a list of python packages
+        :param requirements_file:   path to a python requirements file
+        :param overwrite:           overwrite existing requirements,
+                                    when False (default) will append to existing requirements
+        :return: function object
+        """
+        if isinstance(requirements, str) and mlrun.utils.is_file_path(requirements):
+            # TODO: remove in 1.6.0
+            warnings.warn(
+                "Passing a requirements file path as a string in the 'requirements' argument is deprecated "
+                "and will be removed in 1.6.0, use 'requirements_file' instead",
+                FutureWarning,
+            )
+
+        resolved_requirements = self._resolve_requirements(
+            requirements, requirements_file
+        )
+        requirements = self.requirements or [] if not overwrite else []
+
+        # make sure we don't append the same line twice
+        for requirement in resolved_requirements:
+            if requirement not in requirements:
+                requirements.append(requirement)
+
+        self.requirements = requirements
+
+    @staticmethod
+    def _resolve_requirements(
+        requirements: typing.Union[str, list], requirements_file: str = ""
+    ) -> list:
+        requirements_to_resolve = []
+
+        # handle the requirements_file argument
+        if requirements_file:
+            with open(requirements_file, "r") as fp:
+                requirements_to_resolve.extend(fp.read().splitlines())
+
+        # handle the requirements argument
+        # TODO: remove in 1.6.0, when requirements can only be a list
+        if isinstance(requirements, str):
+            # if it's a file path, read the file and add its content to the list
+            if mlrun.utils.is_file_path(requirements):
+                with open(requirements, "r") as fp:
+                    requirements_to_resolve.extend(fp.read().splitlines())
+            else:
+                # it's a string but not a file path, split it by lines and add it to the list
+                requirements_to_resolve.append(requirements)
+        else:
+            # it's a list, add it to the list
+            requirements_to_resolve.extend(requirements)
+
+        requirements = []
+        for requirement in requirements_to_resolve:
+            # clean redundant leading and trailing whitespaces
+            requirement = requirement.strip()
+
+            # ignore empty lines
+            # ignore comments
+            if not requirement or requirement.startswith("#"):
+                continue
+
+            # ignore inline comments as well
+            inline_comment = requirement.split(" #")
+            if len(inline_comment) > 1:
+                requirement = inline_comment[0].strip()
+
+            requirements.append(requirement)
+
+        return requirements
+
+
+class Notification(ModelObj):
+    """Notification specification"""
+
+    def __init__(
+        self,
+        kind=None,
+        name=None,
+        message=None,
+        severity=None,
+        when=None,
+        condition=None,
+        params=None,
+        status=None,
+        sent_time=None,
+    ):
+        self.kind = kind or mlrun.common.schemas.notification.NotificationKind.slack
+        self.name = name or ""
+        self.message = message or ""
+        self.severity = (
+            severity or mlrun.common.schemas.notification.NotificationSeverity.INFO
+        )
+        self.when = when or ["completed"]
+        self.condition = condition or ""
+        self.params = params or {}
+        self.status = status
+        self.sent_time = sent_time
+
+        self.validate_notification()
+
+    def validate_notification(self):
+        try:
+            mlrun.common.schemas.notification.Notification(**self.to_dict())
+        except pydantic.error_wrappers.ValidationError as exc:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Invalid notification object"
+            ) from exc
+
+    @staticmethod
+    def validate_notification_uniqueness(notifications: List["Notification"]):
+        """Validate that all notifications in the list are unique by name"""
+        names = [notification.name for notification in notifications]
+        if len(names) != len(set(names)):
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Notification names must be unique"
+            )
+
 
 class RunMetadata(ModelObj):
     """Run metadata"""
@@ -492,6 +697,7 @@ class RunSpec(ModelObj):
         allow_empty_resources=None,
         inputs_type_hints=None,
         returns=None,
+        notifications=None,
     ):
         # A dictionary of parsing configurations that will be read from the inputs the user set. The keys are the inputs
         # keys (parameter names) and the values are the type hint given in the input keys after the colon.
@@ -526,6 +732,7 @@ class RunSpec(ModelObj):
         self.verbose = verbose
         self.scrape_metrics = scrape_metrics
         self.allow_empty_resources = allow_empty_resources
+        self._notifications = notifications or []
 
     def to_dict(self, fields=None, exclude=None):
         struct = super().to_dict(fields, exclude=["handler"])
@@ -608,6 +815,9 @@ class RunSpec(ModelObj):
 
         :raise MLRunInvalidArgumentError: In case one of the values in the list is invalid.
         """
+        # This import is located in the method due to circular imports error.
+        from mlrun.package.utils import LogHintUtils
+
         if returns is None:
             self._returns = None
             return
@@ -615,7 +825,7 @@ class RunSpec(ModelObj):
 
         # Validate:
         for log_hint in returns:
-            mlrun.run._parse_log_hint(log_hint=log_hint)
+            LogHintUtils.parse_log_hint(log_hint=log_hint)
 
         # Store the results:
         self._returns = returns
@@ -678,9 +888,22 @@ class RunSpec(ModelObj):
                 return str(self.handler)
         return ""
 
+    @property
+    def notifications(self):
+        return self._notifications
+
+    @notifications.setter
+    def notifications(self, notifications):
+        if isinstance(notifications, list):
+            self._notifications = ObjectList.from_list(Notification, notifications)
+        elif isinstance(notifications, ObjectList):
+            self._notifications = notifications
+        else:
+            raise ValueError("Notifications must be a list")
+
     def extract_type_hints_from_inputs(self):
         """
-        This method extracts the type hints from the inputs keys in the input dictionary.
+        This method extracts the type hints from the input keys in the input dictionary.
 
         As a result, after the method ran the inputs dictionary - a dictionary of parameter names as keys and paths as
         values, will be cleared from type hints and the extracted type hints will be saved in the spec's inputs type
@@ -763,7 +986,7 @@ class RunSpec(ModelObj):
         # Validate correct pattern:
         if input_key.count(":") > 1:
             raise mlrun.errors.MLRunInvalidArgumentError(
-                f"Incorrect input pattern. Inputs keys can have only a single ':' in them to specify the desired type "
+                f"Incorrect input pattern. Input keys can have only a single ':' in them to specify the desired type "
                 f"the input will be parsed as. Given: {input_key}."
             )
 
@@ -790,6 +1013,7 @@ class RunStatus(ModelObj):
         iterations=None,
         ui_url=None,
         reason: str = None,
+        notifications: Dict[str, Notification] = None,
     ):
         self.state = state or "created"
         self.status_text = status_text
@@ -803,6 +1027,7 @@ class RunStatus(ModelObj):
         self.iterations = iterations
         self.ui_url = ui_url
         self.reason = reason
+        self.notifications = notifications or {}
 
 
 class RunTemplate(ModelObj):
@@ -973,6 +1198,20 @@ class RunObject(RunTemplate):
     def status(self, status):
         self._status = self._verify_dict(status, "status", RunStatus)
 
+    @property
+    def error(self) -> str:
+        """error string if failed"""
+        if self.status:
+            if self.status.state != "error":
+                return f"Run state ({self.status.state}) is not in error state"
+            return (
+                self.status.error
+                or self.status.reason
+                or self.status.status_text
+                or "Unknown error"
+            )
+        return ""
+
     def output(self, key):
         """return the value of a specific result or artifact by key"""
         self._outputs_wait_for_completion()
@@ -1144,7 +1383,6 @@ class RunObject(RunTemplate):
                 )
         if logs_enabled and not logs_interval:
             self.logs(watch=False)
-
         if raise_on_failure and state != mlrun.runtimes.constants.RunStates.completed:
             raise mlrun.errors.MLRunRuntimeError(
                 f"task {self.metadata.name} did not complete (state={state})"

mlrun/model_monitoring/__init__.py

@@ -0,0 +1,41 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
+# for backwards compatibility
+
+__all__ = [
+    "ModelEndpoint",
+    "EventFieldType",
+    "EventLiveStats",
+    "EventKeyMetrics",
+    "TimeSeriesTarget",
+    "ModelEndpointTarget",
+    "FileTargetKind",
+    "ProjectSecretKeys",
+    "ModelMonitoringStoreKinds",
+]
+
+from mlrun.common.model_monitoring import (
+    EventFieldType,
+    EventKeyMetrics,
+    EventLiveStats,
+    FileTargetKind,
+    ModelEndpointTarget,
+    ModelMonitoringStoreKinds,
+    ProjectSecretKeys,
+    TimeSeriesTarget,
+)
+
+from .model_endpoint import ModelEndpoint

mlrun/model_monitoring/features_drift_table.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/model_monitoring/helpers.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,18 +13,25 @@
 # limitations under the License.
 #
 import pathlib
+import typing
 
 import sqlalchemy.orm
+from fastapi import Depends
 
 import mlrun
 import mlrun.api.api.utils
 import mlrun.api.crud.secrets
 import mlrun.api.utils.singletons.db
+import mlrun.api.utils.singletons.k8s
+import mlrun.common.model_monitoring as model_monitoring_constants
+import mlrun.common.schemas
 import mlrun.config
 import mlrun.feature_store as fstore
 import mlrun.model_monitoring.stream_processing_fs
 import mlrun.runtimes
 import mlrun.utils.helpers
+import mlrun.utils.model_monitoring
+from mlrun.api.api import deps
 
 _CURRENT_FILE_PATH = pathlib.Path(__file__)
 _STREAM_PROCESSING_FUNCTION_PATH = _CURRENT_FILE_PATH.parent / "stream_processing_fs.py"
@@ -36,16 +43,20 @@ _MONIOTINRG_BATCH_FUNCTION_PATH = (
 def initial_model_monitoring_stream_processing_function(
     project: str,
     model_monitoring_access_key: str,
-    db_session: sqlalchemy.orm.Session,
     tracking_policy: mlrun.utils.model_monitoring.TrackingPolicy,
+    auth_info: mlrun.common.schemas.AuthInfo,
+    parquet_target: str,
 ):
     """
     Initialize model monitoring stream processing function.
 
-    :param project:                     project name.
-    :param model_monitoring_access_key: access key to apply the model monitoring process.
-    :param db_session:                  A session that manages the current dialog with the database.
+    :param project:                     Project name.
+    :param model_monitoring_access_key: Access key to apply the model monitoring process. Please note that in CE
+                                        deployments this parameter will be None.
     :param tracking_policy:             Model monitoring configurations.
+    :param auth_info:                   The auth info of the request.
+    :parquet_target:                    Path to model monitoring parquet file that will be generated by the monitoring
+                                        stream nuclio function.
 
     :return:                            A function object from a mlrun runtime class
 
@@ -54,12 +65,11 @@ def initial_model_monitoring_stream_processing_function(
     # Initialize Stream Processor object
     stream_processor = mlrun.model_monitoring.stream_processing_fs.EventStreamProcessor(
         project=project,
-        model_monitoring_access_key=model_monitoring_access_key,
         parquet_batching_max_events=mlrun.mlconf.model_endpoint_monitoring.parquet_batching_max_events,
+        parquet_target=parquet_target,
+        model_monitoring_access_key=model_monitoring_access_key,
     )
 
-    http_source = mlrun.datastore.sources.HttpSource()
-
     # Create a new serving function for the streaming process
     function = mlrun.code_to_function(
         name="model-monitoring-stream",
@@ -75,46 +85,34 @@ def initial_model_monitoring_stream_processing_function(
     # Set the project to the serving function
     function.metadata.project = project
 
-    # Add v3io stream trigger
-    stream_path = mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
-        project=project, kind="stream"
-    )
-    function.add_v3io_stream_trigger(
-        stream_path=stream_path, name="monitoring_stream_trigger"
-    )
-
-    # Set model monitoring access key for managing permissions
-    function.set_env_from_secret(
-        "MODEL_MONITORING_ACCESS_KEY",
-        mlrun.api.utils.singletons.k8s.get_k8s().get_project_secret_name(project),
-        mlrun.api.crud.secrets.Secrets().generate_client_project_secret_key(
-            mlrun.api.crud.secrets.SecretsClientType.model_monitoring,
-            "MODEL_MONITORING_ACCESS_KEY",
-        ),
+    # Add stream triggers
+    function = _apply_stream_trigger(
+        project=project,
+        function=function,
+        model_monitoring_access_key=model_monitoring_access_key,
+        auth_info=auth_info,
     )
 
+    # Apply feature store run configurations on the serving function
     run_config = fstore.RunConfig(function=function, local=False)
     function.spec.parameters = run_config.parameters
 
-    func = http_source.add_nuclio_trigger(function)
-    func.metadata.credentials.access_key = model_monitoring_access_key
-    func.apply(mlrun.v3io_cred())
-
-    return func
+    return function
 
 
 def get_model_monitoring_batch_function(
     project: str,
     model_monitoring_access_key: str,
     db_session: sqlalchemy.orm.Session,
-    auth_info: mlrun.api.schemas.AuthInfo,
+    auth_info: mlrun.common.schemas.AuthInfo,
     tracking_policy: mlrun.utils.model_monitoring.TrackingPolicy,
 ):
     """
     Initialize model monitoring batch function.
 
     :param project:                     project name.
-    :param model_monitoring_access_key: access key to apply the model monitoring process.
+    :param model_monitoring_access_key: access key to apply the model monitoring process. Please note that in CE
+                                        deployments this parameter will be None.
     :param db_session:                  A session that manages the current dialog with the database.
     :param auth_info:                   The auth info of the request.
     :param tracking_policy:             Model monitoring configurations.
@@ -137,20 +135,107 @@ def get_model_monitoring_batch_function(
     # Set the project to the job function
     function.metadata.project = project
 
+    if not mlrun.mlconf.is_ce_mode():
+        function = _apply_access_key_and_mount_function(
+            project=project,
+            function=function,
+            model_monitoring_access_key=model_monitoring_access_key,
+            auth_info=auth_info,
+        )
+
+    # Enrich runtime with the required configurations
+    mlrun.api.api.utils.apply_enrichment_and_validation_on_function(function, auth_info)
+
+    return function
+
+
+def _apply_stream_trigger(
+    project: str,
+    function: mlrun.runtimes.ServingRuntime,
+    model_monitoring_access_key: str = None,
+    auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
+) -> mlrun.runtimes.ServingRuntime:
+    """Adding stream source for the nuclio serving function. By default, the function has HTTP stream trigger along
+    with another supported stream source that can be either Kafka or V3IO, depends on the stream path schema that is
+    defined under mlrun.mlconf.model_endpoint_monitoring.store_prefixes. Note that if no valid stream path has been
+    provided then the function will have a single HTTP stream source.
+
+    :param project:                     Project name.
+    :param function:                    The serving function object that will be applied with the stream trigger.
+    :param model_monitoring_access_key: Access key to apply the model monitoring stream function when the stream is
+                                        schema is V3IO.
+    :param auth_info:                   The auth info of the request.
+
+    :return: ServingRuntime object with stream trigger.
+    """
+
+    # Get the stream path from the configuration
+    # stream_path = mlrun.mlconf.get_file_target_path(project=project, kind="stream", target="stream")
+    stream_path = mlrun.utils.model_monitoring.get_stream_path(project=project)
+
+    if stream_path.startswith("kafka://"):
+
+        topic, brokers = mlrun.datastore.utils.parse_kafka_url(url=stream_path)
+        # Generate Kafka stream source
+        stream_source = mlrun.datastore.sources.KafkaSource(
+            brokers=brokers,
+            topics=[topic],
+        )
+        function = stream_source.add_nuclio_trigger(function)
+
+    if not mlrun.mlconf.is_ce_mode():
+        function = _apply_access_key_and_mount_function(
+            project=project,
+            function=function,
+            model_monitoring_access_key=model_monitoring_access_key,
+            auth_info=auth_info,
+        )
+        if stream_path.startswith("v3io://"):
+            # Generate V3IO stream trigger
+            function.add_v3io_stream_trigger(
+                stream_path=stream_path, name="monitoring_stream_trigger"
+            )
+    # Add the default HTTP source
+    http_source = mlrun.datastore.sources.HttpSource()
+    function = http_source.add_nuclio_trigger(function)
+
+    return function
+
+
+def _apply_access_key_and_mount_function(
+    project: str,
+    function: typing.Union[
+        mlrun.runtimes.KubejobRuntime, mlrun.runtimes.ServingRuntime
+    ],
+    model_monitoring_access_key: str,
+    auth_info: mlrun.common.schemas.AuthInfo,
+) -> typing.Union[mlrun.runtimes.KubejobRuntime, mlrun.runtimes.ServingRuntime]:
+    """Applying model monitoring access key on the provided function when using V3IO path. In addition, this method
+    mount the V3IO path for the provided function to configure the access to the system files.
+
+    :param project:                     Project name.
+    :param function:                    Model monitoring function object that will be filled with the access key and
+                                        the access to the system files.
+    :param model_monitoring_access_key: Access key to apply the model monitoring stream function when the stream is
+                                        schema is V3IO.
+    :param auth_info:                   The auth info of the request.
+
+    :return: function runtime object with access key and access to system files.
+    """
+
     # Set model monitoring access key for managing permissions
     function.set_env_from_secret(
-        "MODEL_MONITORING_ACCESS_KEY",
-        mlrun.api.utils.singletons.k8s.get_k8s().get_project_secret_name(project),
+        model_monitoring_constants.ProjectSecretKeys.ACCESS_KEY,
+        mlrun.api.utils.singletons.k8s.get_k8s_helper().get_project_secret_name(
+            project
+        ),
         mlrun.api.crud.secrets.Secrets().generate_client_project_secret_key(
             mlrun.api.crud.secrets.SecretsClientType.model_monitoring,
-            "MODEL_MONITORING_ACCESS_KEY",
+            model_monitoring_constants.ProjectSecretKeys.ACCESS_KEY,
         ),
     )
-
-    function.apply(mlrun.mount_v3io())
-
-    # Needs to be a member of the project and have access to project data path
     function.metadata.credentials.access_key = model_monitoring_access_key
+    function.apply(mlrun.mount_v3io())
 
     # Ensure that the auth env vars are set
     mlrun.api.api.utils.ensure_function_has_auth_set(function, auth_info)