mlrun 1.8.0rc18__py3-none-any.whl → 1.8.0rc20__py3-none-any.whl

mlrun/__main__.py

@@ -32,6 +32,7 @@ from tabulate import tabulate
 import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
+import mlrun.utils.helpers
 from mlrun.common.helpers import parse_versioned_object_uri
 from mlrun.runtimes.mounts import auto_mount as auto_mount_modifier
 
@@ -304,6 +305,7 @@ def run(
                 update_in(runtime, "spec.build.code_origin", url_file)
     elif runtime:
         runtime = py_eval(runtime)
+        runtime = mlrun.utils.helpers.as_dict(runtime)
         if not isinstance(runtime, dict):
             print(f"Runtime parameter must be a dict, not {type(runtime)}")
             exit(1)
@@ -515,6 +517,7 @@ def build(
 
     if runtime:
         runtime = py_eval(runtime)
+        runtime = mlrun.utils.helpers.as_dict(runtime)
         if not isinstance(runtime, dict):
             print(f"Runtime parameter must be a dict, not {type(runtime)}")
             exit(1)
@@ -662,6 +665,8 @@ def deploy(
         runtime = py_eval(spec)
     else:
         runtime = {}
+
+    runtime = mlrun.utils.helpers.as_dict(runtime)
     if not isinstance(runtime, dict):
         print(f"Runtime parameter must be a dict, not {type(runtime)}")
         exit(1)

mlrun/common/runtimes/constants.py

@@ -214,6 +214,23 @@ class RunStates:
             RunStates.skipped: mlrun_pipelines.common.models.RunStatuses.skipped,
         }[run_state]
 
+    @staticmethod
+    def pipeline_run_status_to_run_state(pipeline_run_status):
+        if pipeline_run_status not in mlrun_pipelines.common.models.RunStatuses.all():
+            raise ValueError(f"Invalid pipeline run status: {pipeline_run_status}")
+        return {
+            mlrun_pipelines.common.models.RunStatuses.succeeded: RunStates.completed,
+            mlrun_pipelines.common.models.RunStatuses.failed: RunStates.error,
+            mlrun_pipelines.common.models.RunStatuses.running: RunStates.running,
+            mlrun_pipelines.common.models.RunStatuses.pending: RunStates.pending,
+            mlrun_pipelines.common.models.RunStatuses.canceled: RunStates.aborted,
+            mlrun_pipelines.common.models.RunStatuses.canceling: RunStates.aborting,
+            mlrun_pipelines.common.models.RunStatuses.skipped: RunStates.skipped,
+            mlrun_pipelines.common.models.RunStatuses.runtime_state_unspecified: RunStates.unknown,
+            mlrun_pipelines.common.models.RunStatuses.error: RunStates.error,
+            mlrun_pipelines.common.models.RunStatuses.paused: RunStates.unknown,
+        }[pipeline_run_status]
+
 
 # TODO: remove this class in 1.9.0 - use only MlrunInternalLabels
 class RunLabels(enum.Enum):

mlrun/common/schemas/artifact.py

@@ -51,6 +51,12 @@ class ArtifactCategories(mlrun.common.types.StrEnum):
                 True,
             )
 
+    @classmethod
+    def from_kind(cls, kind: str) -> "ArtifactCategories":
+        if kind in [cls.model.value, cls.dataset.value, cls.document.value]:
+            return cls(kind)
+        return cls.other
+
 
 class ArtifactIdentifier(pydantic.v1.BaseModel):
     # artifact kind

mlrun/common/schemas/model_monitoring/__init__.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from .constants import (
+    INTERSECT_DICT_KEYS,
     V3IO_MODEL_MONITORING_DB,
     ApplicationEvent,
     ControllerPolicy,

mlrun/common/schemas/model_monitoring/constants.py

@@ -61,6 +61,7 @@ class ModelEndpointSchema(MonitoringStrEnum):
     STATE = "state"
     MONITORING_MODE = "monitoring_mode"
     FIRST_REQUEST = "first_request"
+    SAMPLING_PERCENTAGE = "sampling_percentage"
 
     # status - operative
     LAST_REQUEST = "last_request"
@@ -137,6 +138,10 @@ class EventFieldType:
     SAMPLE_PARQUET_PATH = "sample_parquet_path"
     TIME = "time"
     TABLE_COLUMN = "table_column"
+    SAMPLING_PERCENTAGE = "sampling_percentage"
+    SAMPLING_RATE = "sampling_rate"
+    ESTIMATED_PREDICTION_COUNT = "estimated_prediction_count"
+    EFFECTIVE_SAMPLE_COUNT = "effective_sample_count"
 
 
 class FeatureSetFeatures(MonitoringStrEnum):
@@ -248,6 +253,12 @@ class ProjectSecretKeys:
         ]
 
 
+class GetEventsFormat(MonitoringStrEnum):
+    SINGLE = "single"
+    SEPARATION = "separation"
+    INTERSECTION = "intersection"
+
+
 class ModelEndpointTargetSchemas(MonitoringStrEnum):
     V3IO = "v3io"
     MYSQL = "mysql"
@@ -448,3 +459,8 @@ FQN_REGEX = re.compile(FQN_PATTERN)
 PROJECT_PATTERN = r"^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$"
 
 MODEL_ENDPOINT_ID_PATTERN = r"^[a-zA-Z0-9_-]+$"
+
+INTERSECT_DICT_KEYS = {
+    ModelEndpointMonitoringMetricType.METRIC: "intersect_metrics",
+    ModelEndpointMonitoringMetricType.RESULT: "intersect_results",
+}

mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -160,6 +160,7 @@ class ModelEndpointStatus(ObjectStatus, ModelEndpointParser):
     state: Optional[str] = "unknown"  # will be updated according to the function state
     first_request: Optional[datetime] = None
     monitoring_mode: Optional[ModelMonitoringMode] = ModelMonitoringMode.disabled
+    sampling_percentage: Optional[float] = 100
 
     # operative
     last_request: Optional[datetime] = None
@@ -177,6 +178,7 @@ class ModelEndpointStatus(ObjectStatus, ModelEndpointParser):
             "monitoring_mode",
             "first_request",
             "last_request",
+            "sampling_percentage",
         ]
 
 
@@ -259,12 +261,12 @@ class ModelEndpointMonitoringMetric(BaseModel):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        self.full_name = _compose_full_name(
+        self.full_name = compose_full_name(
             project=self.project, app=self.app, name=self.name, type=self.type
         )
 
 
-def _compose_full_name(
+def compose_full_name(
     *,
     project: str,
     app: str,

mlrun/config.py

@@ -83,8 +83,8 @@ default_config = {
     "images_to_enrich_registry": "^mlrun/*,python:3.9",
     "kfp_url": "",
     "kfp_ttl": "14400",  # KFP ttl in sec, after that completed PODs will be deleted
-    "kfp_image": "mlrun/mlrun-kfp",  # image to use for KFP runner (defaults to mlrun/mlrun-kfp)
-    "dask_kfp_image": "mlrun/ml-base",  # image to use for dask KFP runner (defaults to mlrun/ml-base)
+    "kfp_image": "mlrun/mlrun-kfp",  # image to use for KFP runner
+    "dask_kfp_image": "mlrun/ml-base",  # image to use for dask KFP runner
     "igz_version": "",  # the version of the iguazio system the API is running on
     "iguazio_api_url": "",  # the url to iguazio api
     "spark_app_image": "",  # image to use for spark operator app runtime

mlrun/db/base.py

@@ -68,6 +68,15 @@ class RunDBInterface(ABC):
     ):
         pass
 
+    def push_pipeline_notifications(
+        self,
+        pipeline_id,
+        project="",
+        notifications=None,
+        timeout=45,
+    ):
+        pass
+
     @abstractmethod
     def read_run(
         self,
@@ -337,6 +346,15 @@ class RunDBInterface(ABC):
     ) -> list[mm_endpoints.ModelEndpointMonitoringMetric]:
         pass
 
+    def get_metrics_by_multiple_endpoints(
+        self,
+        project: str,
+        endpoint_ids: Union[str, list[str]],
+        type: Literal["results", "metrics", "all"] = "all",
+        events_format: mm_constants.GetEventsFormat = mm_constants.GetEventsFormat.SEPARATION,
+    ) -> dict[str, list[mm_endpoints.ModelEndpointMonitoringMetric]]:
+        pass
+
     @abstractmethod
     def delete_project(
         self,

mlrun/db/httpdb.py

@@ -780,9 +780,84 @@ class HTTPRunDB(RunDBInterface):
         )
         if response.status_code == http.HTTPStatus.ACCEPTED:
             background_task = mlrun.common.schemas.BackgroundTask(**response.json())
-            return self._wait_for_background_task_to_reach_terminal_state(
+            background_task = self._wait_for_background_task_to_reach_terminal_state(
+                background_task.metadata.name, project=project
+            )
+            if (
+                background_task.status.state
+                == mlrun.common.schemas.BackgroundTaskState.succeeded
+            ):
+                logger.info(
+                    "Notifications for the run have been pushed",
+                    project=project,
+                    run_id=uid,
+                )
+            elif (
+                background_task.status.state
+                == mlrun.common.schemas.BackgroundTaskState.failed
+            ):
+                logger.error(
+                    "Failed to push run notifications",
+                    project=project,
+                    run_id=uid,
+                    error=background_task.status.error,
+                )
+        return None
+
+    def push_pipeline_notifications(
+        self,
+        pipeline_id,
+        project="",
+        notifications=None,
+        timeout=45,
+    ):
+        """
+        Push notifications for a pipeline.
+
+        :param pipeline_id: Unique ID of the pipeline(KFP).
+        :param project: Project that the run belongs to.
+        :param notifications: List of notifications to push.
+        :returns: :py:class:`~mlrun.common.schemas.BackgroundTask`.
+        """
+        if notifications is None or type(notifications) is not list:
+            raise MLRunInvalidArgumentError(
+                "The 'notifications' parameter must be a list."
+            )
+
+        project = project or config.default_project
+
+        response = self.api_call(
+            "POST",
+            path=f"projects/{project}/pipelines/{pipeline_id}/push-notifications",
+            error="Failed push notifications",
+            body=_as_json([notification.to_dict() for notification in notifications]),
+            timeout=timeout,
+        )
+        if response.status_code == http.HTTPStatus.ACCEPTED:
+            background_task = mlrun.common.schemas.BackgroundTask(**response.json())
+            background_task = self._wait_for_background_task_to_reach_terminal_state(
                 background_task.metadata.name, project=project
             )
+            if (
+                background_task.status.state
+                == mlrun.common.schemas.BackgroundTaskState.succeeded
+            ):
+                logger.info(
+                    "Pipeline notifications have been pushed",
+                    project=project,
+                    pipeline_id=pipeline_id,
+                )
+            elif (
+                background_task.status.state
+                == mlrun.common.schemas.BackgroundTaskState.failed
+            ):
+                logger.error(
+                    "Failed to push pipeline notifications",
+                    project=project,
+                    pipeline_id=pipeline_id,
+                    error=background_task.status.error,
+                )
+
         return None
 
     def read_run(
@@ -3524,6 +3599,48 @@ class HTTPRunDB(RunDBInterface):
             list[mm_endpoints.ModelEndpointMonitoringMetric], monitoring_metrics
         )
 
+    def get_metrics_by_multiple_endpoints(
+        self,
+        project: str,
+        endpoint_ids: Union[str, list[str]],
+        type: Literal["results", "metrics", "all"] = "all",
+        events_format: mm_constants.GetEventsFormat = mm_constants.GetEventsFormat.SEPARATION,
+    ) -> dict[str, list[mm_endpoints.ModelEndpointMonitoringMetric]]:
+        """Get application metrics/results by endpoint id and project.
+
+        :param project:         The name of the project.
+        :param endpoint_ids:    The unique id of the model endpoint. Can be a single id or a list of ids.
+        :param type:            The type of the metrics to return. "all" means "results" and "metrics".
+        :param events_format:   response format:
+
+                                separation: {"mep_id1":[...], "mep_id2":[...]}
+                                intersection {"intersect_metrics":[], "intersect_results":[]}
+        :return: A dictionary of application metrics and/or results for the model endpoints formatted by events_format.
+        """
+        path = f"projects/{project}/model-endpoints/metrics"
+        params = {
+            "type": type,
+            "endpoint-id": endpoint_ids,
+            "events_format": events_format,
+        }
+        error_message = (
+            f"Failed to get model monitoring metrics,"
+            f" endpoint_ids: {endpoint_ids}, project: {project}"
+        )
+        response = self.api_call(
+            mlrun.common.types.HTTPMethod.GET,
+            path,
+            error_message,
+            params=params,
+        )
+        monitoring_metrics_by_endpoint = response.json()
+        parsed_metrics_by_endpoint = {}
+        for endpoint, metrics in monitoring_metrics_by_endpoint.items():
+            parsed_metrics_by_endpoint[endpoint] = parse_obj_as(
+                list[mm_endpoints.ModelEndpointMonitoringMetric], metrics
+            )
+        return parsed_metrics_by_endpoint
+
     def create_user_secrets(
         self,
         user: str,

mlrun/db/nopdb.py

@@ -84,6 +84,15 @@ class NopDB(RunDBInterface):
     ):
         pass
 
+    def push_pipeline_notifications(
+        self,
+        pipeline_id,
+        project="",
+        notifications=None,
+        timeout=45,
+    ):
+        pass
+
     def list_runtime_resources(
         self,
         project: Optional[str] = None,

mlrun/frameworks/_common/model_handler.py

@@ -976,7 +976,6 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
             custom_objects_map_json,
             local_path=custom_objects_map_json,
             artifact_path=self._context.artifact_path,
-            db_key=False,
         )
 
         # Zip the custom objects directory:
@@ -997,7 +996,6 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
             custom_objects_zip,
             local_path=custom_objects_zip,
             artifact_path=self._context.artifact_path,
-            db_key=False,
         )
 
         return artifacts

mlrun/model_monitoring/db/tsdb/base.py

@@ -234,14 +234,14 @@ class TSDBConnector(ABC):
     @abstractmethod
     def get_metrics_metadata(
         self,
-        endpoint_id: str,
+        endpoint_id: typing.Union[str, list[str]],
         start: typing.Optional[datetime] = None,
         end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
         """
-        Fetches distinct metrics metadata from the metrics TSDB table for a specified model endpoint.
+        Fetches distinct metrics metadata from the metrics TSDB table for a specified model endpoints.
 
-        :param endpoint_id:        The model endpoint identifier.
+        :param endpoint_id:        The model endpoint identifier. Can be a single id or a list of ids.
         :param start:              The start time of the query.
         :param end:                The end time of the query.
 
@@ -252,14 +252,14 @@ class TSDBConnector(ABC):
     @abstractmethod
     def get_results_metadata(
         self,
-        endpoint_id: str,
+        endpoint_id: typing.Union[str, list[str]],
         start: typing.Optional[datetime] = None,
         end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
         """
-        Fetches distinct results metadata from the app-results TSDB table for a specified model endpoint.
+        Fetches distinct results metadata from the app-results TSDB table for a specified model endpoints.
 
-        :param endpoint_id:        The model endpoint identifier.
+        :param endpoint_id:        The model endpoint identifier. Can be a single id or a list of ids.
         :param start:              The start time of the query.
         :param end:                The end time of the query.
 
@@ -341,7 +341,7 @@ class TSDBConnector(ABC):
             logger.debug("No metrics", missing_metrics=metrics_without_data.keys())
             grouped = []
         for (app_name, name), sub_df in grouped:
-            full_name = mlrun.model_monitoring.helpers._compose_full_name(
+            full_name = mm_schemas.model_endpoints.compose_full_name(
                 project=project,
                 app=app_name,
                 name=name,
@@ -410,7 +410,7 @@ class TSDBConnector(ABC):
             result_kind = mlrun.model_monitoring.db.tsdb.helpers._get_result_kind(
                 sub_df
             )
-            full_name = mlrun.model_monitoring.helpers._compose_full_name(
+            full_name = mm_schemas.model_endpoints.compose_full_name(
                 project=project, app=app_name, name=name
             )
             try:
@@ -467,6 +467,7 @@ class TSDBConnector(ABC):
 
         :return:        A list of mm metrics objects.
         """
+
         return list(
             map(
                 lambda record: mm_schemas.ModelEndpointMonitoringMetric(
@@ -481,6 +482,113 @@ class TSDBConnector(ABC):
             )
         )
 
+    @staticmethod
+    def df_to_metrics_grouped_dict(
+        *,
+        df: pd.DataFrame,
+        project: str,
+        type: str,
+    ) -> dict[str, list[mm_schemas.ModelEndpointMonitoringMetric]]:
+        """
+        Parse a DataFrame of metrics from the TSDB into a grouped mm metrics objects by endpoint_id.
+
+        :param df:      The DataFrame to parse.
+        :param project: The project name.
+        :param type:    The type of the metrics (either "result" or "metric").
+
+        :return:        A grouped dict of mm metrics/results, using model_endpoints_ids as keys.
+        """
+
+        if df.empty:
+            return {}
+
+        grouped_by_fields = [mm_schemas.WriterEvent.APPLICATION_NAME]
+        if type == "result":
+            name_column = mm_schemas.ResultData.RESULT_NAME
+            grouped_by_fields.append(mm_schemas.ResultData.RESULT_KIND)
+        else:
+            name_column = mm_schemas.MetricData.METRIC_NAME
+
+        grouped_by_fields.append(name_column)
+        # groupby has different behavior for category columns
+        df["endpoint_id"] = df["endpoint_id"].astype(str)
+        grouped_by_df = df.groupby("endpoint_id")
+        grouped_dict = grouped_by_df.apply(
+            lambda group: list(
+                map(
+                    lambda record: mm_schemas.ModelEndpointMonitoringMetric(
+                        project=project,
+                        type=type,
+                        app=record.get(mm_schemas.WriterEvent.APPLICATION_NAME),
+                        name=record.get(name_column),
+                        **{"kind": record.get(mm_schemas.ResultData.RESULT_KIND)}
+                        if type == "result"
+                        else {},
+                    ),
+                    group[grouped_by_fields].to_dict(orient="records"),
+                )
+            )
+        ).to_dict()
+        return grouped_dict
+
+    @staticmethod
+    def df_to_events_intersection_dict(
+        *,
+        df: pd.DataFrame,
+        project: str,
+        type: typing.Union[str, mm_schemas.ModelEndpointMonitoringMetricType],
+    ) -> dict[str, list[mm_schemas.ModelEndpointMonitoringMetric]]:
+        """
+        Parse a DataFrame of metrics from the TSDB into a dict of intersection metrics/results by name and application
+         (and kind in results).
+
+        :param df:      The DataFrame to parse.
+        :param project: The project name.
+        :param type:    The type of the metrics (either "result" or "metric").
+
+        :return:        A dictionary where the key is event type (as defined by `INTERSECT_DICT_KEYS`),
+                        and the value is a list containing the intersect metrics or results across all endpoint IDs.
+
+                        For example:
+                        {
+                            "intersect_metrics": [...]
+                        }
+        """
+        dict_key = mm_schemas.INTERSECT_DICT_KEYS[type]
+        metrics = []
+        if df.empty:
+            return {dict_key: []}
+
+        columns_to_zip = [mm_schemas.WriterEvent.APPLICATION_NAME]
+
+        if type == "result":
+            name_column = mm_schemas.ResultData.RESULT_NAME
+            columns_to_zip.append(mm_schemas.ResultData.RESULT_KIND)
+        else:
+            name_column = mm_schemas.MetricData.METRIC_NAME
+        columns_to_zip.insert(1, name_column)
+
+        # groupby has different behavior for category columns
+        df["endpoint_id"] = df["endpoint_id"].astype(str)
+        df["event_values"] = list(zip(*[df[col] for col in columns_to_zip]))
+        grouped_by_event_values = df.groupby("endpoint_id")["event_values"].apply(set)
+        common_event_values_combinations = set.intersection(*grouped_by_event_values)
+        result_kind = None
+        for data in common_event_values_combinations:
+            application_name, event_name = data[0], data[1]
+            if len(data) > 2:  # in result case
+                result_kind = data[2]
+            metrics.append(
+                mm_schemas.ModelEndpointMonitoringMetric(
+                    project=project,
+                    type=type,
+                    app=application_name,
+                    name=event_name,
+                    kind=result_kind,
+                )
+            )
+        return {dict_key: metrics}
+
     @staticmethod
     def _get_start_end(
         start: typing.Union[datetime, None],

mlrun/model_monitoring/db/tsdb/tdengine/schemas.py

@@ -298,6 +298,8 @@ class Predictions(TDEngineSchema):
             mm_schemas.EventFieldType.TIME: _TDEngineColumn.TIMESTAMP,
             mm_schemas.EventFieldType.LATENCY: _TDEngineColumn.FLOAT,
             mm_schemas.EventKeyMetrics.CUSTOM_METRICS: _TDEngineColumn.BINARY_1000,
+            mm_schemas.EventFieldType.ESTIMATED_PREDICTION_COUNT: _TDEngineColumn.FLOAT,
+            mm_schemas.EventFieldType.EFFECTIVE_SAMPLE_COUNT: _TDEngineColumn.INT,
         }
         tags = {
             mm_schemas.WriterEvent.ENDPOINT_ID: _TDEngineColumn.BINARY_64,