mlrun 1.5.0rc11__py3-none-any.whl → 1.5.0rc13__py3-none-any.whl

mlrun/feature_store/api.py

@@ -975,37 +975,9 @@ def _ingest_with_spark(
             )
 
             df_to_write = df
-
-            # If partitioning by time, add the necessary columns
-            if timestamp_key and "partitionBy" in spark_options:
-                from pyspark.sql.functions import (
-                    dayofmonth,
-                    hour,
-                    minute,
-                    month,
-                    second,
-                    year,
-                )
-
-                time_unit_to_op = {
-                    "year": year,
-                    "month": month,
-                    "day": dayofmonth,
-                    "hour": hour,
-                    "minute": minute,
-                    "second": second,
-                }
-                timestamp_col = df_to_write[timestamp_key]
-                for partition in spark_options["partitionBy"]:
-                    if (
-                        partition not in df_to_write.columns
-                        and partition in time_unit_to_op
-                    ):
-                        op = time_unit_to_op[partition]
-                        df_to_write = df_to_write.withColumn(
-                            partition, op(timestamp_col)
-                        )
-            df_to_write = target.prepare_spark_df(df_to_write, key_columns)
+            df_to_write = target.prepare_spark_df(
+                df_to_write, key_columns, timestamp_key, spark_options
+            )
             if overwrite:
                 df_to_write.write.mode("overwrite").save(**spark_options)
             else:

mlrun/feature_store/feature_vector.py

@@ -631,7 +631,7 @@ class FeatureVector(ModelObj):
             feature_set_fields:  list of field (name, alias) per featureset
         """
         processed_features = {}  # dict of name to (featureset, feature object)
-        feature_set_objects = {}
+        feature_set_objects = self.feature_set_objects or {}
         index_keys = []
         feature_set_fields = collections.defaultdict(list)
         features = copy(self.spec.features)

mlrun/feature_store/retrieval/base.py

@@ -136,7 +136,7 @@ class BaseMerger(abc.ABC):
             order_by=order_by,
         )
 
-    def _write_to_offline_target(self):
+    def _write_to_offline_target(self, timestamp_key=None):
         if self._target:
             is_persistent_vector = self.vector.metadata.name is not None
             if not self._target.path and not is_persistent_vector:
@@ -144,7 +144,12 @@ class BaseMerger(abc.ABC):
                     "target path was not specified"
                 )
             self._target.set_resource(self.vector)
-            size = self._target.write_dataframe(self._result_df)
+            size = self._target.write_dataframe(
+                self._result_df,
+                timestamp_key=timestamp_key
+                if not self._drop_indexes and timestamp_key not in self._drop_columns
+                else None,
+            )
             if is_persistent_vector:
                 target_status = self._target.update_resource_status("ready", size=size)
                 logger.info(f"wrote target: {target_status}")
@@ -361,7 +366,7 @@ class BaseMerger(abc.ABC):
                 )
             self._order_by(order_by_active)
 
-        self._write_to_offline_target()
+        self._write_to_offline_target(timestamp_key=result_timestamp)
         return OfflineVectorResponse(self)
 
     def init_online_vector_service(

mlrun/launcher/remote.py

@@ -89,7 +89,7 @@ class ClientRemoteLauncher(launcher.ClientBaseLauncher):
 
             else:
                 raise mlrun.errors.MLRunRuntimeError(
-                    "function image is not built/ready, set auto_build=True or use .deploy() method first"
+                    "Function image is not built/ready, set auto_build=True or use .deploy() method first"
                 )
 
         if runtime.verbose:
@@ -122,11 +122,11 @@ class ClientRemoteLauncher(launcher.ClientBaseLauncher):
             resp = db.submit_job(run, schedule=schedule)
             if schedule:
                 action = resp.pop("action", "created")
-                logger.info(f"task schedule {action}", **resp)
+                logger.info(f"Task schedule {action}", **resp)
                 return
 
         except (requests.HTTPError, Exception) as err:
-            logger.error(f"got remote run err, {mlrun.errors.err_to_str(err)}")
+            logger.error("Failed remote run", error=mlrun.errors.err_to_str(err))
 
             if isinstance(err, requests.HTTPError):
                 runtime._handle_submit_job_http_error(err)

mlrun/lists.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import warnings
 from copy import copy
 from typing import List
 
@@ -219,6 +220,16 @@ class ArtifactList(list):
         """return as a list of artifact objects"""
         return [dict_to_artifact(artifact) for artifact in self]
 
+    def objects(self) -> List[Artifact]:
+        """return as a list of artifact objects"""
+        warnings.warn(
+            "'objects' is deprecated in 1.3.0 and will be removed in 1.6.0. "
+            "Use 'to_objects' instead.",
+            # TODO: remove in 1.6.0
+            FutureWarning,
+        )
+        return [dict_to_artifact(artifact) for artifact in self]
+
     def dataitems(self) -> List["mlrun.DataItem"]:
         """return as a list of DataItem objects"""
         dataitems = []

mlrun/model_monitoring/__init__.py

@@ -15,7 +15,6 @@
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 # for backwards compatibility
 
-
 from .helpers import get_stream_path
 from .model_endpoint import ModelEndpoint
 from .stores import ModelEndpointStore, ModelEndpointStoreType, get_model_endpoint_store

mlrun/model_monitoring/api.py

@@ -28,9 +28,9 @@ from mlrun.common.schemas.model_monitoring import EventFieldType, ModelMonitorin
 from mlrun.data_types.infer import InferOptions, get_df_stats
 from mlrun.utils import logger
 
+from .batch import VirtualDrift
 from .features_drift_table import FeaturesDriftTablePlot
 from .model_endpoint import ModelEndpoint
-from .model_monitoring_batch import VirtualDrift
 
 # A union of all supported dataset types:
 DatasetType = typing.Union[

mlrun/model_monitoring/application.py

@@ -0,0 +1,313 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import dataclasses
+import json
+from typing import Any, Dict, List, Tuple, Union
+
+import numpy as np
+import pandas as pd
+
+import mlrun.common.helpers
+import mlrun.common.schemas.model_monitoring
+import mlrun.utils.v3io_clients
+from mlrun.datastore import get_stream_pusher
+from mlrun.datastore.targets import ParquetTarget
+from mlrun.model_monitoring.helpers import get_stream_path
+from mlrun.serving.utils import StepToDict
+from mlrun.utils import logger
+
+
+@dataclasses.dataclass
+class ModelMonitoringApplicationResult:
+    """
+    Class representing the result of a custom model monitoring application.
+
+    :param application_name:     (str) Name of the model monitoring application.
+    :param endpoint_id:          (str) ID of the monitored model endpoint.
+    :param schedule_time:        (pd.Timestamp)Timestamp of the monitoring schedule.
+    :param result_name:          (str) Name of the application result.
+    :param result_value:         (float) Value of the application result.
+    :param result_kind:          (ResultKindApp) Kind of application result.
+    :param result_status:        (ResultStatusApp) Status of the application result.
+    :param result_extra_data:    (dict) Extra data associated with the application result.
+
+    """
+
+    application_name: str
+    endpoint_id: str
+    schedule_time: pd.Timestamp
+    result_name: str
+    result_value: float
+    result_kind: mlrun.common.schemas.model_monitoring.constants.ResultKindApp
+    result_status: mlrun.common.schemas.model_monitoring.constants.ResultStatusApp
+    result_extra_data: dict
+
+    def to_dict(self):
+        """
+        Convert the object to a dictionary format suitable for writing.
+
+        :returns:    (dict) Dictionary representation of the result.
+        """
+        return {
+            mlrun.common.schemas.model_monitoring.constants.WriterEvent.APPLICATION_NAME: self.application_name,
+            mlrun.common.schemas.model_monitoring.constants.WriterEvent.ENDPOINT_ID: self.endpoint_id,
+            mlrun.common.schemas.model_monitoring.constants.WriterEvent.SCHEDULE_TIME: self.schedule_time.isoformat(
+                sep=" ", timespec="microseconds"
+            ),
+            mlrun.common.schemas.model_monitoring.constants.WriterEvent.RESULT_NAME: self.result_name,
+            mlrun.common.schemas.model_monitoring.constants.WriterEvent.RESULT_VALUE: self.result_value,
+            mlrun.common.schemas.model_monitoring.constants.WriterEvent.RESULT_KIND: self.result_kind.value,
+            mlrun.common.schemas.model_monitoring.constants.WriterEvent.RESULT_STATUS: self.result_status.value,
+            mlrun.common.schemas.model_monitoring.constants.WriterEvent.RESULT_EXTRA_DATA: json.dumps(
+                self.result_extra_data
+            ),
+        }
+
+
+class ModelMonitoringApplication(StepToDict):
+    """
+    Class representing a model monitoring application. Subclass this to create custom monitoring logic.
+
+    example for very simple costume application::
+        # mlrun: start-code
+        class MyApp(ModelMonitoringApplication):
+
+            def run_application(
+                self,
+                sample_df_stats: pd.DataFrame,
+                feature_stats: pd.DataFrame,
+                sample_df: pd.DataFrame,
+                schedule_time: pd.Timestamp,
+                latest_request: pd.Timestamp,
+                endpoint_id: str,
+                output_stream_uri: str,
+            ) -> typing.Union[ModelMonitoringApplicationResult, typing.List[ModelMonitoringApplicationResult]
+            ]:
+                self.context.log_artifact(TableArtifact("sample_df_stats", df=sample_df_stats))
+                return ModelMonitoringApplicationResult(
+                    self.name,
+                    endpoint_id,
+                    schedule_time,
+                    result_name="data_drift_test",
+                    result_value=0.5,
+                    result_kind=mlrun.common.schemas.model_monitoring.constants.ResultKindApp.data_drift,
+                    result_status = mlrun.common.schemas.model_monitoring.constants.ResultStatusApp.detected,
+                    result_extra_data={})
+
+        # mlrun: end-code
+    """
+
+    kind = "monitoring_application"
+
+    def do(self, event: Dict[str, Any]):
+        """
+        Process the monitoring event and return application results.
+
+        :param event:   (dict) The monitoring event to process.
+        :returns:       (List[ModelMonitoringApplicationResult]) The application results.
+        """
+        resolved_event = self._resolve_event(event)
+        if not (
+            hasattr(self, "context") and isinstance(self.context, mlrun.MLClientCtx)
+        ):
+            self._lazy_init(app_name=resolved_event[0])
+        return self.run_application(*resolved_event)
+
+    def _lazy_init(self, app_name: str):
+        self.context = self._create_context_for_logging(app_name=app_name)
+
+    def run_application(
+        self,
+        application_name: str,
+        sample_df_stats: pd.DataFrame,
+        feature_stats: pd.DataFrame,
+        sample_df: pd.DataFrame,
+        schedule_time: pd.Timestamp,
+        latest_request: pd.Timestamp,
+        endpoint_id: str,
+        output_stream_uri: str,
+    ) -> Union[
+        ModelMonitoringApplicationResult, List[ModelMonitoringApplicationResult]
+    ]:
+        """
+        Implement this method with your custom monitoring logic.
+
+        :param application_name     (str) the app name
+        :param sample_df_stats:     (pd.DataFrame) The new sample distribution DataFrame.
+        :param feature_stats:       (pd.DataFrame) The train sample distribution DataFrame.
+        :param sample_df:           (pd.DataFrame) The new sample DataFrame.
+        :param schedule_time:       (pd.Timestamp) Timestamp of the monitoring schedule.
+        :param latest_request:      (pd.Timestamp) Timestamp of the latest request on this endpoint_id.
+        :param endpoint_id:         (str) ID of the monitored model endpoint
+        :param output_stream_uri:   (str) URI of the output stream for results
+
+        :returns:                   (ModelMonitoringApplicationResult) or
+                                    (List[ModelMonitoringApplicationResult]) of the application results.
+        """
+        raise NotImplementedError
+
+    @staticmethod
+    def _resolve_event(
+        event: Dict[str, Any],
+    ) -> Tuple[
+        str,
+        pd.DataFrame,
+        pd.DataFrame,
+        pd.DataFrame,
+        pd.Timestamp,
+        pd.Timestamp,
+        str,
+        str,
+    ]:
+        """
+        Converting the event into a single tuple that will be be used for passing the event arguments to the running
+        application
+
+        :param event: dictionary with all the incoming data
+
+        :return: A tuple of:
+                     [0] = (str) application name
+                     [1] = (pd.DataFrame) current input statistics
+                     [2] = (pd.DataFrame) train statistics
+                     [3] = (pd.DataFrame) current input data
+                     [4] = (pd.Timestamp) timestamp of batch schedule time
+                     [5] = (pd.Timestamp) timestamp of the latest request
+                     [6] = (str) endpoint id
+                     [7] = (str) output stream uri
+        """
+        return (
+            event[
+                mlrun.common.schemas.model_monitoring.constants.ApplicationEvent.APPLICATION_NAME
+            ],
+            ModelMonitoringApplication._dict_to_histogram(
+                json.loads(
+                    event[
+                        mlrun.common.schemas.model_monitoring.constants.ApplicationEvent.CURRENT_STATS
+                    ]
+                )
+            ),
+            ModelMonitoringApplication._dict_to_histogram(
+                json.loads(
+                    event[
+                        mlrun.common.schemas.model_monitoring.constants.ApplicationEvent.FEATURE_STATS
+                    ]
+                )
+            ),
+            ParquetTarget(
+                path=event[
+                    mlrun.common.schemas.model_monitoring.constants.ApplicationEvent.SAMPLE_PARQUET_PATH
+                ]
+            ).as_df(),
+            pd.Timestamp(
+                event[
+                    mlrun.common.schemas.model_monitoring.constants.ApplicationEvent.SCHEDULE_TIME
+                ]
+            ),
+            pd.Timestamp(
+                event[
+                    mlrun.common.schemas.model_monitoring.constants.ApplicationEvent.LAST_REQUEST
+                ]
+            ),
+            event[
+                mlrun.common.schemas.model_monitoring.constants.ApplicationEvent.ENDPOINT_ID
+            ],
+            event[
+                mlrun.common.schemas.model_monitoring.constants.ApplicationEvent.OUTPUT_STREAM_URI
+            ],
+        )
+
+    @staticmethod
+    def _create_context_for_logging(app_name: str):
+        context = mlrun.get_or_create_ctx(
+            f"{app_name}-logger",
+            upload_artifacts=True,
+            labels={"workflow": "model-monitoring-app-logger"},
+        )
+        return context
+
+    @staticmethod
+    def _dict_to_histogram(histogram_dict: Dict[str, Dict[str, Any]]) -> pd.DataFrame:
+        """
+        Convert histogram dictionary to pandas DataFrame with feature histograms as columns
+
+        :param histogram_dict: Histogram dictionary
+
+        :returns: Histogram dataframe
+        """
+
+        # Create a dictionary with feature histograms as values
+        histograms = {}
+        for feature, stats in histogram_dict.items():
+            if "hist" in stats:
+                # Normalize to probability distribution of each feature
+                histograms[feature] = np.array(stats["hist"][0]) / stats["count"]
+
+        # Convert the dictionary to pandas DataFrame
+        histograms = pd.DataFrame(histograms)
+
+        return histograms
+
+
+class PushToMonitoringWriter(StepToDict):
+    kind = "monitoring_application_stream_pusher"
+
+    def __init__(
+        self,
+        project: str = None,
+        writer_application_name: str = None,
+        stream_uri: str = None,
+        name: str = None,
+    ):
+        """
+        Class for pushing application results to the monitoring writer stream.
+
+        :param project:                     Project name.
+        :param writer_application_name:     Writer application name.
+        :param stream_uri:                  Stream URI for pushing results.
+        :param name:                        Name of the PushToMonitoringWriter
+                                            instance default to PushToMonitoringWriter.
+        """
+        self.project = project
+        self.application_name_to_push = writer_application_name
+        self.stream_uri = stream_uri or get_stream_path(
+            project=self.project, application_name=self.application_name_to_push
+        )
+        self.output_stream = None
+        self.name = name or "PushToMonitoringWriter"
+
+    def do(
+        self,
+        event: Union[
+            ModelMonitoringApplicationResult, List[ModelMonitoringApplicationResult]
+        ],
+    ):
+        """
+        Push application results to the monitoring writer stream.
+
+        :param event: Monitoring result(s) to push.
+        """
+        self._lazy_init()
+        event = event if isinstance(event, List) else [event]
+        for result in event:
+            data = result.to_dict()
+            logger.info(f"Pushing data = {data} \n to stream = {self.stream_uri}")
+            self.output_stream.push([data])
+
+    def _lazy_init(self):
+        if self.output_stream is None:
+            self.output_stream = get_stream_pusher(
+                self.stream_uri,
+            )