mlrun 1.7.0rc18__py3-none-any.whl → 1.7.0rc19__py3-none-any.whl

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -11,13 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import datetime
+
 import typing
+from datetime import datetime
+from io import StringIO
+from typing import Literal, Optional, Union
 
 import pandas as pd
 import v3io_frames.client
 import v3io_frames.errors
-from v3io.dataplane import Client as V3IOClient
 from v3io_frames.frames_pb2 import IGNORE
 
 import mlrun.common.model_monitoring
@@ -25,10 +27,12 @@ import mlrun.common.schemas.model_monitoring as mm_schemas
 import mlrun.feature_store.steps
 import mlrun.utils.v3io_clients
 from mlrun.model_monitoring.db import TSDBConnector
+from mlrun.model_monitoring.helpers import get_invocations_fqn
 from mlrun.utils import logger
 
 _TSDB_BE = "tsdb"
 _TSDB_RATE = "1/s"
+_CONTAINER = "users"
 
 
 class V3IOTSDBConnector(TSDBConnector):
@@ -42,13 +46,11 @@ class V3IOTSDBConnector(TSDBConnector):
     def __init__(
         self,
         project: str,
-        access_key: typing.Optional[str] = None,
-        container: str = "users",
+        container: str = _CONTAINER,
         v3io_framesd: typing.Optional[str] = None,
         create_table: bool = False,
-    ):
+    ) -> None:
         super().__init__(project=project)
-        self.access_key = access_key or mlrun.mlconf.get_v3io_access_key()
 
         self.container = container
 
@@ -56,9 +58,6 @@ class V3IOTSDBConnector(TSDBConnector):
         self._frames_client: v3io_frames.client.ClientBase = (
             self._get_v3io_frames_client(self.container)
         )
-        self._v3io_client: V3IOClient = mlrun.utils.v3io_clients.get_v3io_client(
-            endpoint=mlrun.mlconf.v3io_api,
-        )
 
         self._init_tables_path()
 
@@ -254,7 +253,7 @@ class V3IOTSDBConnector(TSDBConnector):
     ) -> None:
         """Write a single result or metric to TSDB"""
 
-        event[mm_schemas.WriterEvent.END_INFER_TIME] = datetime.datetime.fromisoformat(
+        event[mm_schemas.WriterEvent.END_INFER_TIME] = datetime.fromisoformat(
             event[mm_schemas.WriterEvent.END_INFER_TIME]
         )
         index_cols_base = [
@@ -299,12 +298,9 @@ class V3IOTSDBConnector(TSDBConnector):
         else:
             # Delete all tables
             tables = mm_schemas.V3IOTSDBTables.list()
-        for table in tables:
+        for table_to_delete in tables:
             try:
-                self._frames_client.delete(
-                    backend=mlrun.common.schemas.model_monitoring.TimeSeriesConnector.TSDB,
-                    table=table,
-                )
+                self._frames_client.delete(backend=_TSDB_BE, table=table_to_delete)
             except v3io_frames.errors.DeleteError as e:
                 logger.warning(
                     f"Failed to delete TSDB table '{table}'",
@@ -318,11 +314,7 @@ class V3IOTSDBConnector(TSDBConnector):
         store.rm(tsdb_path, recursive=True)
 
     def get_model_endpoint_real_time_metrics(
-        self,
-        endpoint_id: str,
-        metrics: list[str],
-        start: str,
-        end: str,
+        self, endpoint_id: str, metrics: list[str], start: str, end: str
     ) -> dict[str, list[tuple[str, float]]]:
         """
         Getting real time metrics from the TSDB. There are pre-defined metrics for model endpoints such as
@@ -350,7 +342,7 @@ class V3IOTSDBConnector(TSDBConnector):
         metrics_mapping = {}
 
         try:
-            data = self.get_records(
+            data = self._get_records(
                 table=mm_schemas.V3IOTSDBTables.EVENTS,
                 columns=["endpoint_id", *metrics],
                 filter_query=f"endpoint_id=='{endpoint_id}'",
@@ -375,45 +367,74 @@ class V3IOTSDBConnector(TSDBConnector):
 
         return metrics_mapping
 
-    def get_records(
+    def _get_records(
         self,
         table: str,
-        start: str,
-        end: str,
+        start: Union[datetime, str],
+        end: Union[datetime, str],
         columns: typing.Optional[list[str]] = None,
         filter_query: str = "",
+        interval: typing.Optional[str] = None,
+        agg_funcs: typing.Optional[list] = None,
+        limit: typing.Optional[int] = None,
+        sliding_window_step: typing.Optional[str] = None,
+        **kwargs,
     ) -> pd.DataFrame:
         """
          Getting records from V3IO TSDB data collection.
-        :param table:            Path to the collection to query.
-        :param start:            The start time of the metrics. Can be represented by a string containing an RFC 3339
-                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
-                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, `'d'` = days, and
-                                 `'s'` = seconds), or 0 for the earliest time.
-        :param end:              The end time of the metrics. Can be represented by a string containing an RFC 3339
-                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
-                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, `'d'` = days, and
-                                 `'s'` = seconds), or 0 for the earliest time.
-        :param columns:          Columns to include in the result.
-        :param filter_query:     V3IO filter expression. The expected filter expression includes different conditions,
-                                 divided by ' AND '.
+        :param table:                 Path to the collection to query.
+        :param start:                 The start time of the metrics. Can be represented by a string containing an RFC
+                                      3339 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
+                                      `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, `'d'` = days, and
+                                      `'s'` = seconds), or 0 for the earliest time.
+        :param end:                   The end time of the metrics. Can be represented by a string containing an RFC
+                                      3339 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
+                                      `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, `'d'` = days, and
+                                      `'s'` = seconds), or 0 for the earliest time.
+        :param columns:               Columns to include in the result.
+        :param filter_query:          V3IO filter expression. The expected filter expression includes different
+                                      conditions, divided by ' AND '.
+        :param interval:              The interval to aggregate the data by. Note that if interval is provided,
+                                      agg_funcs must bg provided as well. Provided as a string in the format of '1m',
+                                      '1h', etc.
+        :param agg_funcs:             The aggregation functions to apply on the columns. Note that if `agg_funcs` is
+                                      provided, `interval` must bg provided as well. Provided as a list of strings in
+                                      the format of ['sum', 'avg', 'count', ...].
+        :param limit:                 The maximum number of records to return.
+        :param sliding_window_step:   The time step for which the time window moves forward. Note that if
+                                      `sliding_window_step` is provided, interval must be provided as well. Provided
+                                      as a string in the format of '1m', '1h', etc.
+        :param kwargs:                Additional keyword arguments passed to the read method of frames client.
         :return: DataFrame with the provided attributes from the data collection.
         :raise:  MLRunNotFoundError if the provided table wasn't found.
         """
         if table not in self.tables:
             raise mlrun.errors.MLRunNotFoundError(
-                f"Table '{table}' does not exist in the tables list of the TSDB connector."
+                f"Table '{table}' does not exist in the tables list of the TSDB connector. "
                 f"Available tables: {list(self.tables.keys())}"
             )
-        return self._frames_client.read(
-            backend=mlrun.common.schemas.model_monitoring.TimeSeriesConnector.TSDB,
-            table=self.tables[table],
-            columns=columns,
-            filter=filter_query,
+
+        if agg_funcs:
+            # Frames client expects the aggregators to be a comma-separated string
+            agg_funcs = ",".join(agg_funcs)
+        table_path = self.tables[table]
+        df = self._frames_client.read(
+            backend=_TSDB_BE,
+            table=table_path,
             start=start,
             end=end,
+            columns=columns,
+            filter=filter_query,
+            aggregation_window=interval,
+            aggregators=agg_funcs,
+            step=sliding_window_step,
+            **kwargs,
         )
 
+        if limit:
+            df = df.head(limit)
+        return df
+
     def _get_v3io_source_directory(self) -> str:
         """
         Get the V3IO source directory for the current project. Usually the source directory will
@@ -441,3 +462,169 @@ class V3IOTSDBConnector(TSDBConnector):
             address=mlrun.mlconf.v3io_framesd,
             container=v3io_container,
         )
+
+    def read_metrics_data(
+        self,
+        *,
+        endpoint_id: str,
+        start: datetime,
+        end: datetime,
+        metrics: list[mm_schemas.ModelEndpointMonitoringMetric],
+        type: Literal["metrics", "results"] = "results",
+    ) -> Union[
+        list[
+            Union[
+                mm_schemas.ModelEndpointMonitoringResultValues,
+                mm_schemas.ModelEndpointMonitoringMetricNoData,
+            ],
+        ],
+        list[
+            Union[
+                mm_schemas.ModelEndpointMonitoringMetricValues,
+                mm_schemas.ModelEndpointMonitoringMetricNoData,
+            ],
+        ],
+    ]:
+        """
+        Read metrics OR results from the TSDB and return as a list.
+        Note: the type must match the actual metrics in the `metrics` parameter.
+        If the type is "results", pass only results in the `metrics` parameter.
+        """
+
+        if type == "metrics":
+            table_path = self.tables[mm_schemas.V3IOTSDBTables.METRICS]
+            name = mm_schemas.MetricData.METRIC_NAME
+            df_handler = self.df_to_metrics_values
+        elif type == "results":
+            table_path = self.tables[mm_schemas.V3IOTSDBTables.APP_RESULTS]
+            name = mm_schemas.ResultData.RESULT_NAME
+            df_handler = self.df_to_results_values
+        else:
+            raise ValueError(f"Invalid {type = }")
+
+        query = self._get_sql_query(
+            endpoint_id,
+            [(metric.app, metric.name) for metric in metrics],
+            table_path=table_path,
+            name=name,
+        )
+
+        logger.debug("Querying V3IO TSDB", query=query)
+
+        df: pd.DataFrame = self._frames_client.read(
+            backend=_TSDB_BE,
+            start=start,
+            end=end,
+            query=query,  # the filter argument does not work for this complex condition
+        )
+
+        logger.debug(
+            "Converting a DataFrame to a list of metrics or results values",
+            table=table_path,
+            project=self.project,
+            endpoint_id=endpoint_id,
+            is_empty=df.empty,
+        )
+
+        return df_handler(df=df, metrics=metrics, project=self.project)
+
+    @staticmethod
+    def _get_sql_query(
+        endpoint_id: str,
+        names: list[tuple[str, str]],
+        table_path: str,
+        name: str = mm_schemas.ResultData.RESULT_NAME,
+    ) -> str:
+        """Get the SQL query for the results/metrics table"""
+        with StringIO() as query:
+            query.write(
+                f"SELECT * FROM '{table_path}' "
+                f"WHERE {mm_schemas.WriterEvent.ENDPOINT_ID}='{endpoint_id}'"
+            )
+            if names:
+                query.write(" AND (")
+
+                for i, (app_name, result_name) in enumerate(names):
+                    sub_cond = (
+                        f"({mm_schemas.WriterEvent.APPLICATION_NAME}='{app_name}' "
+                        f"AND {name}='{result_name}')"
+                    )
+                    if i != 0:  # not first sub condition
+                        query.write(" OR ")
+                    query.write(sub_cond)
+
+                query.write(")")
+
+            query.write(";")
+            return query.getvalue()
+
+    def read_predictions(
+        self,
+        *,
+        endpoint_id: str,
+        start: Union[datetime, str],
+        end: Union[datetime, str],
+        aggregation_window: Optional[str] = None,
+        agg_funcs: Optional[list[str]] = None,
+        limit: Optional[int] = None,
+    ) -> Union[
+        mm_schemas.ModelEndpointMonitoringMetricNoData,
+        mm_schemas.ModelEndpointMonitoringMetricValues,
+    ]:
+        if (agg_funcs and not aggregation_window) or (
+            aggregation_window and not agg_funcs
+        ):
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "both or neither of `aggregation_window` and `agg_funcs` must be provided"
+            )
+        df = self._get_records(
+            table=mm_schemas.FileTargetKind.PREDICTIONS,
+            start=start,
+            end=end,
+            columns=[mm_schemas.EventFieldType.LATENCY],
+            filter_query=f"endpoint_id=='{endpoint_id}'",
+            interval=aggregation_window,
+            agg_funcs=agg_funcs,
+            limit=limit,
+            sliding_window_step=aggregation_window,
+        )
+
+        full_name = get_invocations_fqn(self.project)
+
+        if df.empty:
+            return mm_schemas.ModelEndpointMonitoringMetricNoData(
+                full_name=full_name,
+                type=mm_schemas.ModelEndpointMonitoringMetricType.METRIC,
+            )
+
+        latency_column = (
+            f"{agg_funcs[0]}({mm_schemas.EventFieldType.LATENCY})"
+            if agg_funcs
+            else mm_schemas.EventFieldType.LATENCY
+        )
+
+        return mm_schemas.ModelEndpointMonitoringMetricValues(
+            full_name=full_name,
+            values=list(
+                zip(
+                    df.index,
+                    df[latency_column],
+                )
+            ),  # pyright: ignore[reportArgumentType]
+        )
+
+    def read_prediction_metric_for_endpoint_if_exists(
+        self, endpoint_id: str
+    ) -> Optional[mm_schemas.ModelEndpointMonitoringMetric]:
+        # Read just one record, because we just want to check if there is any data for this endpoint_id
+        predictions = self.read_predictions(
+            endpoint_id=endpoint_id, start="0", end="now", limit=1
+        )
+        if predictions:
+            return mm_schemas.ModelEndpointMonitoringMetric(
+                project=self.project,
+                app=mm_schemas.SpecialApps.MLRUN_INFRA,
+                type=mm_schemas.ModelEndpointMonitoringMetricType.METRIC,
+                name=mm_schemas.PredictionsQueryConstants.INVOCATIONS,
+                full_name=get_invocations_fqn(self.project),
+            )

mlrun/model_monitoring/helpers.py

@@ -24,6 +24,10 @@ import mlrun.common.schemas
 from mlrun.common.schemas.model_monitoring import (
     EventFieldType,
 )
+from mlrun.common.schemas.model_monitoring.model_endpoints import (
+    ModelEndpointMonitoringMetricType,
+    _compose_full_name,
+)
 from mlrun.model_monitoring.model_endpoint import ModelEndpoint
 from mlrun.utils import logger
 
@@ -292,3 +296,12 @@ def get_default_result_instance_fqn(model_endpoint_id: str) -> str:
         mm_constants.HistogramDataDriftApplicationConstants.NAME,
         mm_constants.HistogramDataDriftApplicationConstants.GENERAL_RESULT_NAME,
     )
+
+
+def get_invocations_fqn(project: str) -> str:
+    return _compose_full_name(
+        project=project,
+        app=mm_constants.SpecialApps.MLRUN_INFRA,
+        name=mm_constants.PredictionsQueryConstants.INVOCATIONS,
+        type=ModelEndpointMonitoringMetricType.METRIC,
+    )

mlrun/model_monitoring/writer.py

@@ -24,6 +24,7 @@ from mlrun.common.schemas.model_monitoring.constants import (
     HistogramDataDriftApplicationConstants,
     MetricData,
     ResultData,
+    ResultKindApp,
     ResultStatusApp,
     WriterEvent,
     WriterEventKind,
@@ -117,9 +118,13 @@ class ModelMonitoringWriter(StepToDict):
         )
         self._endpoints_records = {}
 
-    @staticmethod
     def _generate_event_on_drift(
-        entity_id: str, drift_status: str, event_value: dict, project_name: str
+        self,
+        entity_id: str,
+        result_status: int,
+        event_value: dict,
+        project_name: str,
+        result_kind: int,
     ) -> None:
         logger.info("Sending an event")
         entity = mlrun.common.schemas.alert.EventEntities(
@@ -127,16 +132,35 @@ class ModelMonitoringWriter(StepToDict):
             project=project_name,
             ids=[entity_id],
         )
-        event_kind = (
-            alert_objects.EventKind.DATA_DRIFT_DETECTED
-            if drift_status == ResultStatusApp.detected.value
-            else alert_objects.EventKind.DATA_DRIFT_SUSPECTED
+
+        event_kind = self._generate_alert_event_kind(
+            result_status=result_status, result_kind=result_kind
         )
+
         event_data = mlrun.common.schemas.Event(
-            kind=event_kind, entity=entity, value_dict=event_value
+            kind=alert_objects.EventKind(value=event_kind),
+            entity=entity,
+            value_dict=event_value,
         )
         mlrun.get_run_db().generate_event(event_kind, event_data)
 
+    @staticmethod
+    def _generate_alert_event_kind(
+        result_kind: int, result_status: int
+    ) -> alert_objects.EventKind:
+        """Generate the required Event Kind format for the alerting system"""
+        if result_kind == ResultKindApp.custom.value:
+            # Custom kind is represented as an anomaly detection
+            event_kind = "mm_app_anomaly"
+        else:
+            event_kind = ResultKindApp(value=result_kind).name
+
+        if result_status == ResultStatusApp.detected.value:
+            event_kind = f"{event_kind}_detected"
+        else:
+            event_kind = f"{event_kind}_suspected"
+        return alert_objects.EventKind(value=event_kind)
+
     @staticmethod
     def _reconstruct_event(event: _RawEvent) -> tuple[_AppResultEvent, WriterEventKind]:
         """
@@ -209,14 +233,15 @@ class ModelMonitoringWriter(StepToDict):
                 "result_value": event[ResultData.RESULT_VALUE],
             }
             self._generate_event_on_drift(
-                get_result_instance_fqn(
+                entity_id=get_result_instance_fqn(
                     event[WriterEvent.ENDPOINT_ID],
                     event[WriterEvent.APPLICATION_NAME],
                     event[ResultData.RESULT_NAME],
                 ),
-                event[ResultData.RESULT_STATUS],
-                event_value,
-                self.project,
+                result_status=event[ResultData.RESULT_STATUS],
+                event_value=event_value,
+                project_name=self.project,
+                result_kind=event[ResultData.RESULT_KIND],
             )
 
         if (

mlrun/projects/operations.py

@@ -18,6 +18,7 @@ from typing import Optional, Union
 from mlrun_pipelines.models import PipelineNodeWrapper
 
 import mlrun
+import mlrun.common.constants as mlrun_constants
 from mlrun.utils import hub_prefix
 
 from .pipelines import enrich_function_object, pipeline_context
@@ -190,7 +191,9 @@ def run_function(
         local = pipeline_context.is_run_local(local)
         task.metadata.labels = task.metadata.labels or labels or {}
         if pipeline_context.workflow_id:
-            task.metadata.labels["workflow"] = pipeline_context.workflow_id
+            task.metadata.labels[mlrun_constants.MLRunInternalLabels.workflow] = (
+                pipeline_context.workflow_id
+            )
         if function.kind == "local":
             command, function = mlrun.run.load_func_code(function)
             function.spec.command = command
@@ -225,9 +228,9 @@ def run_function(
 class BuildStatus:
     """returned status from build operation"""
 
-    def __init__(self, ready, outputs={}, function=None):
+    def __init__(self, ready, outputs=None, function=None):
         self.ready = ready
-        self.outputs = outputs
+        self.outputs = outputs or {}
         self.function = function
 
     def after(self, step):
@@ -340,9 +343,9 @@ def build_function(
 class DeployStatus:
     """returned status from deploy operation"""
 
-    def __init__(self, state, outputs={}, function=None):
+    def __init__(self, state, outputs=None, function=None):
         self.state = state
-        self.outputs = outputs
+        self.outputs = outputs or {}
         self.function = function
 
     def after(self, step):

mlrun/projects/pipelines.py

@@ -26,6 +26,7 @@ from kfp.compiler import compiler
 from mlrun_pipelines.helpers import new_pipe_metadata
 
 import mlrun
+import mlrun.common.runtimes.constants
 import mlrun.common.schemas
 import mlrun.utils.notifications
 from mlrun.errors import err_to_str
@@ -371,7 +372,7 @@ class _PipelineRunStatus:
         engine: type["_PipelineRunner"],
         project: "mlrun.projects.MlrunProject",
         workflow: WorkflowSpec = None,
-        state: str = "",
+        state: mlrun_pipelines.common.models.RunStatuses = "",
         exc: Exception = None,
     ):
         """
@@ -479,6 +480,7 @@ class _PipelineRunner(abc.ABC):
         timeout=None,
         expected_statuses=None,
         notifiers: mlrun.utils.notifications.CustomNotificationPusher = None,
+        **kwargs,
     ):
         pass
 
@@ -610,6 +612,7 @@ class _KFPRunner(_PipelineRunner):
         timeout=None,
         expected_statuses=None,
         notifiers: mlrun.utils.notifications.CustomNotificationPusher = None,
+        **kwargs,
     ):
         if timeout is None:
             timeout = 60 * 60
@@ -733,6 +736,7 @@ class _LocalRunner(_PipelineRunner):
         timeout=None,
         expected_statuses=None,
         notifiers: mlrun.utils.notifications.CustomNotificationPusher = None,
+        **kwargs,
     ):
         pass
 
@@ -860,7 +864,7 @@ class _RemoteRunner(_PipelineRunner):
             )
             state = mlrun_pipelines.common.models.RunStatuses.failed
         else:
-            state = mlrun_pipelines.common.models.RunStatuses.succeeded
+            state = mlrun_pipelines.common.models.RunStatuses.running
             project.notifiers.push_pipeline_start_message(
                 project.metadata.name,
             )
@@ -877,24 +881,47 @@ class _RemoteRunner(_PipelineRunner):
     @staticmethod
     def get_run_status(
         project,
-        run,
+        run: _PipelineRunStatus,
         timeout=None,
         expected_statuses=None,
         notifiers: mlrun.utils.notifications.CustomNotificationPusher = None,
+        inner_engine: type[_PipelineRunner] = None,
     ):
-        # ignore notifiers, as they are handled by the remote pipeline notifications,
-        # so overriding with CustomNotificationPusher with empty list of notifiers
-        state, had_errors, text = _KFPRunner.get_run_status(
-            project,
-            run,
-            timeout,
-            expected_statuses,
-            notifiers=mlrun.utils.notifications.CustomNotificationPusher([]),
-        )
+        inner_engine = inner_engine or _KFPRunner
+        if inner_engine.engine == _KFPRunner.engine:
+            # ignore notifiers for remote notifications, as they are handled by the remote pipeline notifications,
+            # so overriding with CustomNotificationPusher with empty list of notifiers or only local notifiers
+            local_project_notifiers = list(
+                set(mlrun.utils.notifications.NotificationTypes.local()).intersection(
+                    set(project.notifiers.notifications.keys())
+                )
+            )
+            notifiers = mlrun.utils.notifications.CustomNotificationPusher(
+                local_project_notifiers
+            )
+            return _KFPRunner.get_run_status(
+                project,
+                run,
+                timeout,
+                expected_statuses,
+                notifiers=notifiers,
+            )
 
-        # indicate the pipeline status since we don't push the notifications in the remote runner
-        logger.info(text)
-        return state, had_errors, text
+        elif inner_engine.engine == _LocalRunner.engine:
+            mldb = mlrun.db.get_run_db(secrets=project._secrets)
+            pipeline_runner_run = mldb.read_run(run.run_id, project=project.name)
+            pipeline_runner_run = mlrun.run.RunObject.from_dict(pipeline_runner_run)
+            pipeline_runner_run.logs(db=mldb)
+            pipeline_runner_run.refresh()
+            run._state = mlrun.common.runtimes.constants.RunStates.run_state_to_pipeline_run_status(
+                pipeline_runner_run.status.state
+            )
+            run._exc = pipeline_runner_run.status.error
+
+        else:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Unsupported inner runner engine: {inner_engine.engine}"
+            )
 
 
 def create_pipeline(project, pipeline, functions, secrets=None, handler=None):