mlrun 1.5.0rc12__py3-none-any.whl → 1.5.0rc13__py3-none-any.whl

mlrun/api/crud/model_monitoring/deployment.py

@@ -25,21 +25,27 @@ import mlrun.api.utils.scheduler
 import mlrun.api.utils.singletons.db
 import mlrun.api.utils.singletons.k8s
 import mlrun.common.schemas.model_monitoring
+import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.model_monitoring.stream_processing
 import mlrun.model_monitoring.tracking_policy
 from mlrun import feature_store as fstore
 from mlrun.api.api import deps
 from mlrun.api.crud.model_monitoring.helpers import Seconds, seconds2minutes
+from mlrun.model_monitoring.writer import ModelMonitoringWriter
 from mlrun.utils import logger
 
 _MODEL_MONITORING_COMMON_PATH = pathlib.Path(__file__).parents[3] / "model_monitoring"
 _STREAM_PROCESSING_FUNCTION_PATH = (
     _MODEL_MONITORING_COMMON_PATH / "stream_processing.py"
 )
-_MONITORING_BATCH_FUNCTION_PATH = (
-    _MODEL_MONITORING_COMMON_PATH / "model_monitoring_batch.py"
+_MONITORING_ORIGINAL_BATCH_FUNCTION_PATH = _MODEL_MONITORING_COMMON_PATH / "batch.py"
+
+_MONITORING_APPLICATION_BATCH_FUNCTION_PATH = (
+    _MODEL_MONITORING_COMMON_PATH / "batch_application_handler.py"
 )
 
+_MONITORING_WRITER_FUNCTION_PATH = _MODEL_MONITORING_COMMON_PATH / "writer.py"
+
 
 class MonitoringDeployment:
     def __init__(
@@ -50,8 +56,11 @@ class MonitoringDeployment:
         self._parquet_batching_max_events = parquet_batching_max_events
         self._max_parquet_save_interval = max_parquet_save_interval
         """
-        Initialize a MonitoringDeployment object, which handles the deployment of both model monitoring stream nuclio
-        function and the scheduled batch drift job.
+        Initialize a MonitoringDeployment object, which handles the deployment & scheduling of:
+         1. model monitoring stream
+         2. model monitoring batch
+         3. model monitoring batch application
+         4. model monitoring writer
 
         :param parquet_batching_max_events: Maximum number of events that will be used for writing the monitoring
                                             parquet by the monitoring stream function.
@@ -91,7 +100,25 @@ class MonitoringDeployment:
             auth_info=auth_info,
             tracking_policy=tracking_policy,
             tracking_offset=Seconds(self._max_parquet_save_interval),
+            function_name=mm_constants.MonitoringFunctionNames.BATCH,
         )
+        if tracking_policy.application_batch:
+            self.deploy_model_monitoring_batch_processing(
+                project=project,
+                model_monitoring_access_key=model_monitoring_access_key,
+                db_session=db_session,
+                auth_info=auth_info,
+                tracking_policy=tracking_policy,
+                tracking_offset=Seconds(self._max_parquet_save_interval),
+                function_name=mm_constants.MonitoringFunctionNames.BATCH_APPLICATION,
+            )
+            self.deploy_model_monitoring_writer_application(
+                project=project,
+                model_monitoring_access_key=model_monitoring_access_key,
+                db_session=db_session,
+                auth_info=auth_info,
+                tracking_policy=tracking_policy,
+            )
 
     def deploy_model_monitoring_stream_processing(
         self,
@@ -154,7 +181,9 @@ class MonitoringDeployment:
         fn.metadata.labels = {"type": "model-monitoring-stream"}
 
         mlrun.api.api.endpoints.functions._build_function(
-            db_session=db_session, auth_info=auth_info, function=fn
+            db_session=db_session,
+            auth_info=auth_info,
+            function=fn,
         )
 
     def deploy_model_monitoring_batch_processing(
@@ -167,12 +196,14 @@ class MonitoringDeployment:
         with_schedule: bool = True,
         overwrite: bool = False,
         tracking_offset: Seconds = Seconds(0),
+        function_name: str = mm_constants.MonitoringFunctionNames.BATCH,
     ) -> typing.Union[mlrun.runtimes.kubejob.KubejobRuntime, None]:
         """
-        Deploying model monitoring batch job. The goal of this job is to identify drift in the data
-        based on the latest batch of events. By default, this job is executed on the hour every hour.
-        Note that if the monitoring batch job was already deployed then you will either have to pass overwrite=True or
-        to delete the old monitoring batch job before deploying a new one.
+        Deploying model monitoring batch job or model monitoring batch application job.
+        The goal of this job is to identify drift in the data based on the latest batch of events. By default,
+        this job is executed on the hour every hour.
+        Note that if this job was already deployed then you will either have to pass overwrite=True or
+        to delete the old job before deploying a new one.
 
         :param project:                     The name of the project.
         :param model_monitoring_access_key: Access key to apply the model monitoring process.
@@ -182,14 +213,23 @@ class MonitoringDeployment:
         :param with_schedule:               If true, submit a scheduled batch drift job.
         :param overwrite:                   If true, overwrite the existing model monitoring batch job.
         :param tracking_offset:             Offset for the tracking policy (for synchronization with the stream)
+        :param function_name:               model-monitoring-batch or model-monitoring-application-batch
+                                            indicates witch one to deploy.
 
         :return: Model monitoring batch job as a runtime function.
         """
-
+        job_valid_names = [
+            mm_constants.MonitoringFunctionNames.BATCH,
+            mm_constants.MonitoringFunctionNames.BATCH_APPLICATION,
+        ]
+        if function_name not in job_valid_names:
+            raise mlrun.errors.MLRunRuntimeError(
+                f"Model Monitoring batch job can be only within {job_valid_names}"
+            )
         fn = None
         if not overwrite:
             logger.info(
-                "Checking if model monitoring batch processing function is already deployed",
+                f"Checking if {function_name.replace('-',' ')} processing function is already deployed",
                 project=project,
             )
 
@@ -198,17 +238,18 @@ class MonitoringDeployment:
             try:
                 fn = mlrun.api.crud.Functions().get_function(
                     db_session=db_session,
-                    name="model-monitoring-batch",
+                    name=function_name,
                     project=project,
                 )
                 logger.info(
-                    "Detected model monitoring batch processing function already deployed",
+                    f"Detected {function_name.replace('-',' ')} processing function already deployed",
                     project=project,
                 )
 
             except mlrun.errors.MLRunNotFoundError:
                 logger.info(
-                    "Deploying monitoring batch processing function ", project=project
+                    f"Deploying {function_name.replace('-',' ')} processing function ",
+                    project=project,
                 )
 
         if not fn:
@@ -219,6 +260,7 @@ class MonitoringDeployment:
                 db_session=db_session,
                 auth_info=auth_info,
                 tracking_policy=tracking_policy,
+                function_name=function_name,
             )
 
             # Get the function uri
@@ -230,16 +272,16 @@ class MonitoringDeployment:
                         mlrun.api.utils.scheduler.Scheduler().get_schedule(
                             db_session=db_session,
                             project=project,
-                            name="model-monitoring-batch",
+                            name=function_name,
                         )
                         logger.info(
-                            "Already deployed monitoring batch scheduled job function ",
+                            f"Already deployed {function_name.replace('-',' ')} scheduled job function ",
                             project=project,
                         )
                         return
                     except mlrun.errors.MLRunNotFoundError:
                         logger.info(
-                            "Deploying monitoring batch scheduled job function ",
+                            f"Deploying {function_name.replace('-',' ')} scheduled job function ",
                             project=project,
                         )
                 # Submit batch scheduled job
@@ -250,9 +292,68 @@ class MonitoringDeployment:
                     auth_info=auth_info,
                     tracking_policy=tracking_policy,
                     tracking_offset=tracking_offset,
+                    function_name=function_name,
                 )
         return fn
 
+    def deploy_model_monitoring_writer_application(
+        self,
+        project,
+        model_monitoring_access_key,
+        db_session,
+        auth_info,
+        tracking_policy,
+    ):
+        """
+        Deploying model monitoring writer real time nuclio function. The goal of this real time function is
+        to write all the monitoring application result to the databases. It is triggered by those applications.
+        It processes and writes the result to the databases.
+
+        :param project:                     The name of the project.
+        :param model_monitoring_access_key: Access key to apply the model monitoring process.
+        :param db_session:                  A session that manages the current dialog with the database.
+        :param auth_info:                   The auth info of the request.
+        :param tracking_policy:             Model monitoring configurations.
+        """
+
+        logger.info(
+            "Checking if model monitoring writer is already deployed",
+            project=project,
+        )
+        try:
+            # validate that the model monitoring stream has not yet been deployed
+            mlrun.runtimes.function.get_nuclio_deploy_status(
+                name=mm_constants.MonitoringFunctionNames.WRITER,
+                project=project,
+                tag="",
+                auth_info=auth_info,
+            )
+            logger.info(
+                "Detected model monitoring writer processing function already deployed",
+                project=project,
+            )
+            return
+        except mlrun.errors.MLRunNotFoundError:
+            logger.info(
+                "Deploying model monitoring writer processing function", project=project
+            )
+
+        fn = self._initial_model_monitoring_writer_function(
+            project=project,
+            model_monitoring_access_key=model_monitoring_access_key,
+            tracking_policy=tracking_policy,
+            auth_info=auth_info,
+        )
+
+        # Adding label to the function - will be used to identify the stream pod
+        fn.metadata.labels = {"type": "model-monitoring-writer"}
+
+        mlrun.api.api.endpoints.functions._build_function(
+            db_session=db_session,
+            auth_info=auth_info,
+            function=fn,
+        )
+
     def _initial_model_monitoring_stream_processing_function(
         self,
         project: str,
@@ -323,6 +424,7 @@ class MonitoringDeployment:
         db_session: sqlalchemy.orm.Session,
         auth_info: mlrun.common.schemas.AuthInfo,
         tracking_policy: mlrun.model_monitoring.tracking_policy.TrackingPolicy,
+        function_name: str = "model-monitoring-batch",
     ):
         """
         Initialize model monitoring batch function.
@@ -333,16 +435,21 @@ class MonitoringDeployment:
         :param db_session:                  A session that manages the current dialog with the database.
         :param auth_info:                   The auth info of the request.
         :param tracking_policy:             Model monitoring configurations.
-
+        :param function_name:               model-monitoring-batch or model-monitoring-application-batch
+                                            indicates witch one to create.
         :return:                            A function object from a mlrun runtime class
 
         """
-
+        filename = (
+            str(_MONITORING_ORIGINAL_BATCH_FUNCTION_PATH)
+            if function_name == "model-monitoring-batch"
+            else str(_MONITORING_APPLICATION_BATCH_FUNCTION_PATH)
+        )
         # Create job function runtime for the model monitoring batch
         function: mlrun.runtimes.KubejobRuntime = mlrun.code_to_function(
-            name="model-monitoring-batch",
+            name=function_name,
             project=project,
-            filename=str(_MONITORING_BATCH_FUNCTION_PATH),
+            filename=filename,
             kind="job",
             image=tracking_policy.default_batch_image,
             handler="handler",
@@ -358,6 +465,7 @@ class MonitoringDeployment:
                 function=function,
                 model_monitoring_access_key=model_monitoring_access_key,
                 auth_info=auth_info,
+                function_name=function_name,
             )
 
         # Enrich runtime with the required configurations
@@ -375,6 +483,7 @@ class MonitoringDeployment:
         auth_info: mlrun.common.schemas.AuthInfo,
         tracking_policy: mlrun.model_monitoring.tracking_policy.TrackingPolicy,
         tracking_offset: Seconds = Seconds(0),
+        function_name: str = "model-monitoring-batch",
     ):
         """
         Create a new scheduled monitoring batch job analysis based on the model-monitoring-batch function that has
@@ -392,7 +501,7 @@ class MonitoringDeployment:
 
         function_uri = function_uri.replace("db://", "")
 
-        task = mlrun.new_task(name="model-monitoring-batch", project=project)
+        task = mlrun.new_task(name=function_name, project=project)
         task.spec.function = function_uri
 
         # Apply batching interval params
@@ -423,7 +532,8 @@ class MonitoringDeployment:
         }
 
         logger.info(
-            "Deploying model monitoring batch processing function", project=project
+            f"Deploying {function_name.replace('-',' ')} processing function",
+            project=project,
         )
 
         # Add job schedule policy (every hour by default)
@@ -437,6 +547,7 @@ class MonitoringDeployment:
         function: mlrun.runtimes.ServingRuntime,
         model_monitoring_access_key: str = None,
         auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
+        function_name: str = None,
     ) -> mlrun.runtimes.ServingRuntime:
         """Adding stream source for the nuclio serving function. By default, the function has HTTP stream trigger along
         with another supported stream source that can be either Kafka or V3IO, depends on the stream path schema that is
@@ -448,13 +559,17 @@ class MonitoringDeployment:
         :param model_monitoring_access_key: Access key to apply the model monitoring stream function when the stream is
                                             schema is V3IO.
         :param auth_info:                   The auth info of the request.
+        :param function_name:             the name of the function that be applied with the stream trigger,
+                                            None for model_monitoring_stream
 
         :return: ServingRuntime object with stream trigger.
         """
 
         # Get the stream path from the configuration
         # stream_path = mlrun.mlconf.get_file_target_path(project=project, kind="stream", target="stream")
-        stream_path = mlrun.api.crud.model_monitoring.get_stream_path(project=project)
+        stream_path = mlrun.api.crud.model_monitoring.get_stream_path(
+            project=project, application_name=function_name
+        )
 
         if stream_path.startswith("kafka://"):
             topic, brokers = mlrun.datastore.utils.parse_kafka_url(url=stream_path)
@@ -471,11 +586,18 @@ class MonitoringDeployment:
                 function=function,
                 model_monitoring_access_key=model_monitoring_access_key,
                 auth_info=auth_info,
+                function_name=function_name,
             )
             if stream_path.startswith("v3io://"):
                 # Generate V3IO stream trigger
                 function.add_v3io_stream_trigger(
-                    stream_path=stream_path, name="monitoring_stream_trigger"
+                    stream_path=stream_path,
+                    name="monitoring_stream_trigger"
+                    if function_name is None
+                    else f"monitoring_{function_name}_trigger",
+                    access_key=model_monitoring_access_key
+                    if function_name != mm_constants.MonitoringFunctionNames.STREAM
+                    else None,
                 )
         # Add the default HTTP source
         http_source = mlrun.datastore.sources.HttpSource()
@@ -491,6 +613,7 @@ class MonitoringDeployment:
         ],
         model_monitoring_access_key: str,
         auth_info: mlrun.common.schemas.AuthInfo,
+        function_name: str = None,
     ) -> typing.Union[mlrun.runtimes.KubejobRuntime, mlrun.runtimes.ServingRuntime]:
         """Applying model monitoring access key on the provided function when using V3IO path. In addition, this method
         mount the V3IO path for the provided function to configure the access to the system files.
@@ -505,23 +628,70 @@ class MonitoringDeployment:
         :return: function runtime object with access key and access to system files.
         """
 
-        # Set model monitoring access key for managing permissions
-        function.set_env_from_secret(
-            mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ACCESS_KEY,
-            mlrun.api.utils.singletons.k8s.get_k8s_helper().get_project_secret_name(
-                project
-            ),
-            mlrun.api.crud.secrets.Secrets().generate_client_project_secret_key(
-                mlrun.api.crud.secrets.SecretsClientType.model_monitoring,
+        if function_name in mm_constants.MonitoringFunctionNames.all():
+            # Set model monitoring access key for managing permissions
+            function.set_env_from_secret(
                 mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ACCESS_KEY,
-            ),
+                mlrun.api.utils.singletons.k8s.get_k8s_helper().get_project_secret_name(
+                    project
+                ),
+                mlrun.api.crud.secrets.Secrets().generate_client_project_secret_key(
+                    mlrun.api.crud.secrets.SecretsClientType.model_monitoring,
+                    mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ACCESS_KEY,
+                ),
+            )
+
+            function.metadata.credentials.access_key = model_monitoring_access_key
+            function.apply(mlrun.v3io_cred())
+
+            # Ensure that the auth env vars are set
+            mlrun.api.api.utils.ensure_function_has_auth_set(function, auth_info)
+        return function
+
+    def _initial_model_monitoring_writer_function(
+        self, project, model_monitoring_access_key, tracking_policy, auth_info
+    ):
+        """
+        Initialize model monitoring writer function.
+
+        :param project:                     Project name.
+        :param model_monitoring_access_key: Access key to apply the model monitoring process. Please note that in CE
+                                            deployments this parameter will be None.
+        :param tracking_policy:             Model monitoring configurations.
+        :param auth_info:                   The auth info of the request.
+
+        :return:                            A function object from a mlrun runtime class
+
+        """
+
+        # Create a new serving function for the streaming process
+        function = mlrun.code_to_function(
+            name=mm_constants.MonitoringFunctionNames.WRITER,
+            project=project,
+            filename=str(_MONITORING_WRITER_FUNCTION_PATH),
+            kind="serving",
+            image=tracking_policy.stream_image,
         )
 
-        function.metadata.credentials.access_key = model_monitoring_access_key
-        function.apply(mlrun.mount_v3io())
+        # Create writer monitoring serving graph
+        graph = function.set_topology("flow")
+        graph.to(ModelMonitoringWriter(name="king")).respond()  # writer
+
+        # Set the project to the serving function
+        function.metadata.project = project
 
-        # Ensure that the auth env vars are set
-        mlrun.api.api.utils.ensure_function_has_auth_set(function, auth_info)
+        # Add stream triggers
+        function = self._apply_stream_trigger(
+            project=project,
+            function=function,
+            model_monitoring_access_key=model_monitoring_access_key,
+            auth_info=auth_info,
+            function_name=mm_constants.MonitoringFunctionNames.WRITER,
+        )
+
+        # Apply feature store run configurations on the serving function
+        run_config = fstore.RunConfig(function=function, local=False)
+        function.spec.parameters = run_config.parameters
 
         return function
 

mlrun/api/crud/model_monitoring/helpers.py

@@ -112,7 +112,9 @@ def get_access_key(auth_info: mlrun.common.schemas.AuthInfo):
 
 
 def get_monitoring_parquet_path(
-    db_session: sqlalchemy.orm.Session, project: str
+    db_session: sqlalchemy.orm.Session,
+    project: str,
+    kind: str = mlrun.common.schemas.model_monitoring.FileTargetKind.PARQUET,
 ) -> str:
     """Get model monitoring parquet target for the current project. The parquet target path is based on the
     project artifact path. If project artifact path is not defined, the parquet target path will be based on MLRun
@@ -121,6 +123,7 @@ def get_monitoring_parquet_path(
     :param db_session: A session that manages the current dialog with the database. Will be used in this function
                        to get the project record from DB.
     :param project:    Project name.
+    :param kind:       indicate the kind of the parquet path, can be either stream_parquet or stream_controller_parquet
 
     :return:           Monitoring parquet target path.
     """
@@ -133,27 +136,37 @@ def get_monitoring_parquet_path(
     # Generate monitoring parquet path value
     parquet_path = mlrun.mlconf.get_model_monitoring_file_target_path(
         project=project,
-        kind=mlrun.common.schemas.model_monitoring.FileTargetKind.PARQUET,
+        kind=kind,
         target="offline",
         artifact_path=artifact_path,
     )
     return parquet_path
 
 
-def get_stream_path(project: str = None):
-    """Get stream path from the project secret. If wasn't set, take it from the system configurations"""
+def get_stream_path(project: str = None, application_name: str = None):
+    """
+    Get stream path from the project secret. If wasn't set, take it from the system configurations
+
+    :param project:             Project name.
+    :param application_name:    Application name, None for model_monitoring_stream.
+
+    :return:                    Monitoring stream path to the relevant application.
+    """
 
     stream_uri = mlrun.api.crud.secrets.Secrets().get_project_secret(
         project=project,
         provider=mlrun.common.schemas.secret.SecretProviderName.kubernetes,
         allow_secrets_from_k8s=True,
-        secret_key=mlrun.common.schemas.model_monitoring.ProjectSecretKeys.STREAM_PATH,
+        secret_key=mlrun.common.schemas.model_monitoring.ProjectSecretKeys.STREAM_PATH
+        if application_name is None
+        else "",
     ) or mlrun.mlconf.get_model_monitoring_file_target_path(
         project=project,
         kind=mlrun.common.schemas.model_monitoring.FileTargetKind.STREAM,
         target="online",
+        application_name=application_name,
     )
 
     return mlrun.common.model_monitoring.helpers.parse_monitoring_stream_path(
-        stream_uri=stream_uri, project=project
+        stream_uri=stream_uri, project=project, application_name=application_name
     )

mlrun/api/crud/model_monitoring/model_endpoints.py

@@ -72,6 +72,12 @@ class ModelEndpoints:
                 )
             )
 
+            mlrun.utils.helpers.verify_field_of_type(
+                field_name="model_endpoint.spec.model_uri",
+                field_value=model_obj,
+                expected_type=mlrun.artifacts.ModelArtifact,
+            )
+
             # Get stats from model object if not found in model endpoint object
             if not model_endpoint.status.feature_stats and hasattr(
                 model_obj, "feature_stats"
@@ -238,6 +244,12 @@ class ModelEndpoints:
                         name=feature.name, value_type=feature.value_type
                     )
                 )
+            for feature in model_obj.spec.outputs:
+                feature_set.add_feature(
+                    mlrun.feature_store.Feature(
+                        name=feature.name, value_type=feature.value_type
+                    )
+                )
         # Check if features can be found within the feature vector
         elif model_obj.spec.feature_vector:
             _, name, _, tag, _ = mlrun.utils.helpers.parse_artifact_uri(
@@ -267,7 +279,8 @@ class ModelEndpoints:
         )
 
         parquet_target = mlrun.datastore.targets.ParquetTarget(
-            mlrun.common.schemas.model_monitoring.FileTargetKind.PARQUET, parquet_path
+            mlrun.common.schemas.model_monitoring.FileTargetKind.PARQUET,
+            parquet_path,
         )
         driver = mlrun.datastore.targets.get_target_driver(parquet_target, feature_set)
 

mlrun/api/db/sqldb/db.py

@@ -304,7 +304,9 @@ class SQLDB(DBInterface):
         project = project or config.default_project
         run = self._get_run(session, uid, project, iter)
         if not run:
-            raise mlrun.errors.MLRunNotFoundError(f"Run {uid}:{project} not found")
+            raise mlrun.errors.MLRunNotFoundError(
+                f"Run uid {uid} of project {project} not found"
+            )
         return run.struct
 
     def list_runs(

mlrun/api/utils/builder.py

@@ -673,10 +673,8 @@ def build_runtime(
     base_image: str = (
         build.base_image or runtime.spec.image or config.default_base_image
     )
-    enriched_base_image = mlrun.utils.enrich_image_url(
-        base_image,
-        client_version,
-        client_python_version,
+    enriched_base_image = runtime.full_image_path(
+        base_image, client_version, client_python_version
     )
     mlrun.utils.logger.info(
         "Building runtime image",

mlrun/common/model_monitoring/helpers.py

@@ -64,20 +64,34 @@ def parse_model_endpoint_store_prefix(store_prefix: str):
     return endpoint, container, path
 
 
-def parse_monitoring_stream_path(stream_uri: str, project: str):
+def parse_monitoring_stream_path(
+    stream_uri: str, project: str, application_name: str = None
+):
     if stream_uri.startswith("kafka://"):
         if "?topic" in stream_uri:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "Custom kafka topic is not allowed"
             )
         # Add topic to stream kafka uri
-        stream_uri += f"?topic=monitoring_stream_{project}"
+        if application_name is None:
+            stream_uri += f"?topic=monitoring_stream_{project}"
+        else:
+            stream_uri += f"?topic=monitoring_stream_{project}_{application_name}"
 
     elif stream_uri.startswith("v3io://") and mlrun.mlconf.is_ce_mode():
         # V3IO is not supported in CE mode, generating a default http stream path
-        stream_uri = mlrun.mlconf.model_endpoint_monitoring.default_http_sink.format(
-            project=project
-        )
+        if application_name is None:
+            stream_uri = (
+                mlrun.mlconf.model_endpoint_monitoring.default_http_sink.format(
+                    project=project
+                )
+            )
+        else:
+            stream_uri = (
+                mlrun.mlconf.model_endpoint_monitoring.default_http_sink_app.format(
+                    project=project, application_name=application_name
+                )
+            )
     return stream_uri