mlrun 1.7.0rc14__py3-none-any.whl → 1.7.0rc15__py3-none-any.whl

mlrun/launcher/client.py

@@ -71,7 +71,7 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
     ):
         run.metadata.labels["kind"] = runtime.kind
         mlrun.runtimes.utils.enrich_run_labels(
-            run.metadata.labels, [mlrun.runtimes.constants.RunLabels.owner]
+            run.metadata.labels, [mlrun.common.runtimes.constants.RunLabels.owner]
         )
         if run.spec.output_path:
             run.spec.output_path = run.spec.output_path.replace(

mlrun/lists.py

@@ -21,7 +21,7 @@ import mlrun.frameworks
 from .artifacts import Artifact, dict_to_artifact
 from .config import config
 from .render import artifacts_to_html, runs_to_html
-from .utils import flatten, get_artifact_target, get_in, is_legacy_artifact
+from .utils import flatten, get_artifact_target, get_in
 
 list_header = [
     "project",
@@ -184,7 +184,7 @@ class ArtifactList(list):
             "uri": ["uri", "uri"],
         }
         for artifact in self:
-            fields_index = 0 if is_legacy_artifact(artifact) else 1
+            fields_index = 1
             row = [get_in(artifact, v[fields_index], "") for k, v in head.items()]
             artifact_uri = dict_to_artifact(artifact).uri
             last_index = len(row) - 1

mlrun/model.py

@@ -33,7 +33,6 @@ from .utils import (
     dict_to_json,
     dict_to_yaml,
     get_artifact_target,
-    is_legacy_artifact,
     logger,
     template_artifact_path,
 )
@@ -1435,11 +1434,14 @@ class RunObject(RunTemplate):
             unknown_error = ""
             if (
                 self.status.state
-                in mlrun.runtimes.constants.RunStates.abortion_states()
+                in mlrun.common.runtimes.constants.RunStates.abortion_states()
             ):
                 unknown_error = "Run was aborted"
 
-            elif self.status.state in mlrun.runtimes.constants.RunStates.error_states():
+            elif (
+                self.status.state
+                in mlrun.common.runtimes.constants.RunStates.error_states()
+            ):
                 unknown_error = "Unknown error"
 
             return (
@@ -1477,7 +1479,7 @@ class RunObject(RunTemplate):
             outputs = {k: v for k, v in self.status.results.items()}
         if self.status.artifacts:
             for a in self.status.artifacts:
-                key = a["key"] if is_legacy_artifact(a) else a["metadata"]["key"]
+                key = a["metadata"]["key"]
                 outputs[key] = get_artifact_target(a, self.metadata.project)
         return outputs
 
@@ -1520,7 +1522,10 @@ class RunObject(RunTemplate):
 
     def state(self):
         """current run state"""
-        if self.status.state in mlrun.runtimes.constants.RunStates.terminal_states():
+        if (
+            self.status.state
+            in mlrun.common.runtimes.constants.RunStates.terminal_states()
+        ):
             return self.status.state
         self.refresh()
         return self.status.state or "unknown"
@@ -1582,7 +1587,7 @@ class RunObject(RunTemplate):
         last_pull_log_time = None
         logs_enabled = show_logs is not False
         state = self.state()
-        if state not in mlrun.runtimes.constants.RunStates.terminal_states():
+        if state not in mlrun.common.runtimes.constants.RunStates.terminal_states():
             logger.info(
                 f"run {self.metadata.name} is not completed yet, waiting for it to complete",
                 current_state=state,
@@ -1592,7 +1597,8 @@ class RunObject(RunTemplate):
             if (
                 logs_enabled
                 and logs_interval
-                and state not in mlrun.runtimes.constants.RunStates.terminal_states()
+                and state
+                not in mlrun.common.runtimes.constants.RunStates.terminal_states()
                 and (
                     last_pull_log_time is None
                     or (datetime.now() - last_pull_log_time).seconds > logs_interval
@@ -1601,7 +1607,7 @@ class RunObject(RunTemplate):
                 last_pull_log_time = datetime.now()
                 state, offset = self.logs(watch=False, offset=offset)
 
-            if state in mlrun.runtimes.constants.RunStates.terminal_states():
+            if state in mlrun.common.runtimes.constants.RunStates.terminal_states():
                 if logs_enabled and logs_interval:
                     self.logs(watch=False, offset=offset)
                 break
@@ -1613,7 +1619,10 @@ class RunObject(RunTemplate):
                 )
         if logs_enabled and not logs_interval:
             self.logs(watch=False)
-        if raise_on_failure and state != mlrun.runtimes.constants.RunStates.completed:
+        if (
+            raise_on_failure
+            and state != mlrun.common.runtimes.constants.RunStates.completed
+        ):
             raise mlrun.errors.MLRunRuntimeError(
                 f"Task {self.metadata.name} did not complete (state={state})"
             )

mlrun/model_monitoring/api.py

@@ -22,9 +22,10 @@ import pandas as pd
 
 import mlrun.artifacts
 import mlrun.common.helpers
-import mlrun.common.schemas.model_monitoring.constants as mm_consts
+import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.feature_store
 import mlrun.model_monitoring.application
+import mlrun.model_monitoring.applications as mm_app
 import mlrun.serving
 from mlrun.data_types.infer import InferOptions, get_df_stats
 from mlrun.utils import datetime_now, logger
@@ -48,7 +49,7 @@ def get_or_create_model_endpoint(
     sample_set_statistics: dict[str, typing.Any] = None,
     drift_threshold: float = None,
     possible_drift_threshold: float = None,
-    monitoring_mode: mm_consts.ModelMonitoringMode = mm_consts.ModelMonitoringMode.disabled,
+    monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.disabled,
     db_session=None,
 ) -> ModelEndpoint:
     """
@@ -128,7 +129,7 @@ def record_results(
     context: typing.Optional[mlrun.MLClientCtx] = None,
     infer_results_df: typing.Optional[pd.DataFrame] = None,
     sample_set_statistics: typing.Optional[dict[str, typing.Any]] = None,
-    monitoring_mode: mm_consts.ModelMonitoringMode = mm_consts.ModelMonitoringMode.enabled,
+    monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.enabled,
     # Deprecated arguments:
     drift_threshold: typing.Optional[float] = None,
     possible_drift_threshold: typing.Optional[float] = None,
@@ -282,7 +283,7 @@ def _model_endpoint_validations(
     # drift and possible drift thresholds
     if drift_threshold:
         current_drift_threshold = model_endpoint.spec.monitor_configuration.get(
-            mm_consts.EventFieldType.DRIFT_DETECTED_THRESHOLD,
+            mm_constants.EventFieldType.DRIFT_DETECTED_THRESHOLD,
             mlrun.mlconf.model_endpoint_monitoring.drift_thresholds.default.drift_detected,
         )
         if current_drift_threshold != drift_threshold:
@@ -293,7 +294,7 @@ def _model_endpoint_validations(
 
     if possible_drift_threshold:
         current_possible_drift_threshold = model_endpoint.spec.monitor_configuration.get(
-            mm_consts.EventFieldType.POSSIBLE_DRIFT_THRESHOLD,
+            mm_constants.EventFieldType.POSSIBLE_DRIFT_THRESHOLD,
             mlrun.mlconf.model_endpoint_monitoring.drift_thresholds.default.possible_drift,
         )
         if current_possible_drift_threshold != possible_drift_threshold:
@@ -332,14 +333,14 @@ def write_monitoring_df(
         )
 
     # Modify the DataFrame to the required structure that will be used later by the monitoring batch job
-    if mm_consts.EventFieldType.TIMESTAMP not in infer_results_df.columns:
+    if mm_constants.EventFieldType.TIMESTAMP not in infer_results_df.columns:
         # Initialize timestamp column with the current time
-        infer_results_df[mm_consts.EventFieldType.TIMESTAMP] = infer_datetime
+        infer_results_df[mm_constants.EventFieldType.TIMESTAMP] = infer_datetime
 
     # `endpoint_id` is the monitoring feature set entity and therefore it should be defined as the df index before
     # the ingest process
-    infer_results_df[mm_consts.EventFieldType.ENDPOINT_ID] = endpoint_id
-    infer_results_df.set_index(mm_consts.EventFieldType.ENDPOINT_ID, inplace=True)
+    infer_results_df[mm_constants.EventFieldType.ENDPOINT_ID] = endpoint_id
+    infer_results_df.set_index(mm_constants.EventFieldType.ENDPOINT_ID, inplace=True)
 
     monitoring_feature_set.ingest(source=infer_results_df, overwrite=False)
 
@@ -355,7 +356,7 @@ def _generate_model_endpoint(
     sample_set_statistics: dict[str, typing.Any],
     drift_threshold: float,
     possible_drift_threshold: float,
-    monitoring_mode: mm_consts.ModelMonitoringMode = mm_consts.ModelMonitoringMode.disabled,
+    monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.disabled,
 ) -> ModelEndpoint:
     """
     Write a new model endpoint record.
@@ -394,11 +395,11 @@ def _generate_model_endpoint(
     model_endpoint.spec.model_class = "drift-analysis"
     if drift_threshold:
         model_endpoint.spec.monitor_configuration[
-            mm_consts.EventFieldType.DRIFT_DETECTED_THRESHOLD
+            mm_constants.EventFieldType.DRIFT_DETECTED_THRESHOLD
         ] = drift_threshold
     if possible_drift_threshold:
         model_endpoint.spec.monitor_configuration[
-            mm_consts.EventFieldType.POSSIBLE_DRIFT_THRESHOLD
+            mm_constants.EventFieldType.POSSIBLE_DRIFT_THRESHOLD
         ] = possible_drift_threshold
 
     model_endpoint.spec.monitoring_mode = monitoring_mode
@@ -589,7 +590,10 @@ def _create_model_monitoring_function_base(
     project: str,
     func: typing.Union[str, None] = None,
     application_class: typing.Union[
-        str, mlrun.model_monitoring.application.ModelMonitoringApplicationBase, None
+        str,
+        mlrun.model_monitoring.application.ModelMonitoringApplicationBase,
+        mm_app.ModelMonitoringApplicationBaseV2,
+        None,
     ] = None,
     name: typing.Optional[str] = None,
     image: typing.Optional[str] = None,
@@ -602,6 +606,20 @@ def _create_model_monitoring_function_base(
     Note: this is an internal API only.
     This function does not set the labels or mounts v3io.
     """
+    if isinstance(
+        application_class,
+        mlrun.model_monitoring.application.ModelMonitoringApplicationBase,
+    ):
+        warnings.warn(
+            "The `ModelMonitoringApplicationBase` class is deprecated from version 1.7.0, "
+            "please use `ModelMonitoringApplicationBaseV2`. It will be removed in 1.9.0.",
+            FutureWarning,
+        )
+    if name in mm_constants.MonitoringFunctionNames.list():
+        raise mlrun.errors.MLRunInvalidArgumentError(
+            f"An application cannot have the following names: "
+            f"{mm_constants.MonitoringFunctionNames.list()}"
+        )
     if func is None:
         func = ""
     func_obj = typing.cast(
@@ -618,14 +636,19 @@ def _create_model_monitoring_function_base(
         ),
     )
     graph = func_obj.set_topology(mlrun.serving.states.StepKinds.flow)
+    prepare_step = graph.to(
+        class_name="mlrun.model_monitoring.applications._application_steps._PrepareMonitoringEvent",
+        name="PrepareMonitoringEvent",
+        application_name=name,
+    )
     if isinstance(application_class, str):
-        first_step = graph.to(class_name=application_class, **application_kwargs)
+        app_step = prepare_step.to(class_name=application_class, **application_kwargs)
     else:
-        first_step = graph.to(class_name=application_class)
-    first_step.to(
-        class_name="mlrun.model_monitoring.application.PushToMonitoringWriter",
+        app_step = prepare_step.to(class_name=application_class)
+    app_step.to(
+        class_name="mlrun.model_monitoring.applications._application_steps._PushToMonitoringWriter",
         name="PushToMonitoringWriter",
         project=project,
-        writer_application_name=mm_consts.MonitoringFunctionNames.WRITER,
+        writer_application_name=mm_constants.MonitoringFunctionNames.WRITER,
     ).respond()
     return func_obj

mlrun/model_monitoring/application.py

@@ -12,308 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import dataclasses
-import json
-import re
-from abc import ABC, abstractmethod
-from typing import Any, Optional, Union, cast
-
-import numpy as np
-import pandas as pd
-
-import mlrun.common.helpers
-import mlrun.common.model_monitoring.helpers
-import mlrun.common.schemas.model_monitoring.constants as mm_constant
-import mlrun.utils.v3io_clients
-from mlrun.datastore import get_stream_pusher
-from mlrun.datastore.targets import ParquetTarget
-from mlrun.model_monitoring.helpers import get_stream_path
-from mlrun.serving.utils import StepToDict
-from mlrun.utils import logger
-
-
-@dataclasses.dataclass
-class ModelMonitoringApplicationResult:
-    """
-    Class representing the result of a custom model monitoring application.
-
-    :param name:           (str) Name of the application result. This name must be
-                            unique for each metric in a single application
-                            (name must be of the format [a-zA-Z_][a-zA-Z0-9_]*).
-    :param value:          (float) Value of the application result.
-    :param kind:           (ResultKindApp) Kind of application result.
-    :param status:         (ResultStatusApp) Status of the application result.
-    :param extra_data:     (dict) Extra data associated with the application result.
-    """
-
-    name: str
-    value: float
-    kind: mm_constant.ResultKindApp
-    status: mm_constant.ResultStatusApp
-    extra_data: dict = dataclasses.field(default_factory=dict)
-
-    def __post_init__(self):
-        pat = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*")
-        if not re.fullmatch(pat, self.name):
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "Attribute name must be of the format [a-zA-Z_][a-zA-Z0-9_]*"
-            )
-
-    def to_dict(self):
-        """
-        Convert the object to a dictionary format suitable for writing.
-
-        :returns:    (dict) Dictionary representation of the result.
-        """
-        return {
-            mm_constant.WriterEvent.RESULT_NAME: self.name,
-            mm_constant.WriterEvent.RESULT_VALUE: self.value,
-            mm_constant.WriterEvent.RESULT_KIND: self.kind,
-            mm_constant.WriterEvent.RESULT_STATUS: self.status,
-            mm_constant.WriterEvent.RESULT_EXTRA_DATA: json.dumps(self.extra_data),
-        }
-
-
-class ModelMonitoringApplicationBase(StepToDict, ABC):
-    """
-    A base class for a model monitoring application.
-    Inherit from this class to create a custom model monitoring application.
-
-    example for very simple custom application::
-        # mlrun: start-code
-        class MyApp(ApplicationBase):
-            def do_tracking(
-                self,
-                sample_df_stats: mlrun.common.model_monitoring.helpers.FeatureStats,
-                feature_stats: mlrun.common.model_monitoring.helpers.FeatureStats,
-                start_infer_time: pd.Timestamp,
-                end_infer_time: pd.Timestamp,
-                schedule_time: pd.Timestamp,
-                latest_request: pd.Timestamp,
-                endpoint_id: str,
-                output_stream_uri: str,
-            ) -> ModelMonitoringApplicationResult:
-                self.context.log_artifact(
-                    TableArtifact(
-                        "sample_df_stats", df=self.dict_to_histogram(sample_df_stats)
-                    )
-                )
-                return ModelMonitoringApplicationResult(
-                    name="data_drift_test",
-                    value=0.5,
-                    kind=mm_constant.ResultKindApp.data_drift,
-                    status=mm_constant.ResultStatusApp.detected,
-                )
-
-
-        # mlrun: end-code
-    """
-
-    kind = "monitoring_application"
-
-    def do(
-        self, event: dict[str, Any]
-    ) -> tuple[list[ModelMonitoringApplicationResult], dict]:
-        """
-        Process the monitoring event and return application results.
-
-        :param event:   (dict) The monitoring event to process.
-        :returns:       (list[ModelMonitoringApplicationResult], dict) The application results
-                        and the original event for the application.
-        """
-        resolved_event = self._resolve_event(event)
-        if not (
-            hasattr(self, "context") and isinstance(self.context, mlrun.MLClientCtx)
-        ):
-            self._lazy_init(app_name=resolved_event[0])
-        results = self.do_tracking(*resolved_event)
-        results = results if isinstance(results, list) else [results]
-        return results, event
-
-    def _lazy_init(self, app_name: str):
-        self.context = cast(
-            mlrun.MLClientCtx, self._create_context_for_logging(app_name=app_name)
-        )
-
-    @abstractmethod
-    def do_tracking(
-        self,
-        application_name: str,
-        sample_df_stats: mlrun.common.model_monitoring.helpers.FeatureStats,
-        feature_stats: mlrun.common.model_monitoring.helpers.FeatureStats,
-        sample_df: pd.DataFrame,
-        start_infer_time: pd.Timestamp,
-        end_infer_time: pd.Timestamp,
-        latest_request: pd.Timestamp,
-        endpoint_id: str,
-        output_stream_uri: str,
-    ) -> Union[
-        ModelMonitoringApplicationResult, list[ModelMonitoringApplicationResult]
-    ]:
-        """
-        Implement this method with your custom monitoring logic.
-
-        :param application_name:         (str) the app name
-        :param sample_df_stats:         (FeatureStats) The new sample distribution dictionary.
-        :param feature_stats:           (FeatureStats) The train sample distribution dictionary.
-        :param sample_df:               (pd.DataFrame) The new sample DataFrame.
-        :param start_infer_time:        (pd.Timestamp) Start time of the monitoring schedule.
-        :param end_infer_time:          (pd.Timestamp) End time of the monitoring schedule.
-        :param latest_request:          (pd.Timestamp) Timestamp of the latest request on this endpoint_id.
-        :param endpoint_id:             (str) ID of the monitored model endpoint
-        :param output_stream_uri:       (str) URI of the output stream for results
-
-        :returns:                       (ModelMonitoringApplicationResult) or
-                                        (list[ModelMonitoringApplicationResult]) of the application results.
-        """
-        raise NotImplementedError
-
-    @classmethod
-    def _resolve_event(
-        cls,
-        event: dict[str, Any],
-    ) -> tuple[
-        str,
-        mlrun.common.model_monitoring.helpers.FeatureStats,
-        mlrun.common.model_monitoring.helpers.FeatureStats,
-        pd.DataFrame,
-        pd.Timestamp,
-        pd.Timestamp,
-        pd.Timestamp,
-        str,
-        str,
-    ]:
-        """
-        Converting the event into a single tuple that will be used for passing the event arguments to the running
-        application
-
-        :param event: dictionary with all the incoming data
-
-        :return: A tuple of:
-                     [0] = (str) application name
-                     [1] = (dict) current input statistics
-                     [2] = (dict) train statistics
-                     [3] = (pd.DataFrame) current input data
-                     [4] = (pd.Timestamp) start time of the monitoring schedule
-                     [5] = (pd.Timestamp) end time of the monitoring schedule
-                     [6] = (pd.Timestamp) timestamp of the latest request
-                     [7] = (str) endpoint id
-                     [8] = (str) output stream uri
-        """
-        start_time = pd.Timestamp(event[mm_constant.ApplicationEvent.START_INFER_TIME])
-        end_time = pd.Timestamp(event[mm_constant.ApplicationEvent.END_INFER_TIME])
-        return (
-            event[mm_constant.ApplicationEvent.APPLICATION_NAME],
-            json.loads(event[mm_constant.ApplicationEvent.CURRENT_STATS]),
-            json.loads(event[mm_constant.ApplicationEvent.FEATURE_STATS]),
-            ParquetTarget(
-                path=event[mm_constant.ApplicationEvent.SAMPLE_PARQUET_PATH]
-            ).as_df(
-                start_time=start_time,
-                end_time=end_time,
-                time_column=mm_constant.FeatureSetFeatures.time_stamp(),
-            ),
-            start_time,
-            end_time,
-            pd.Timestamp(event[mm_constant.ApplicationEvent.LAST_REQUEST]),
-            event[mm_constant.ApplicationEvent.ENDPOINT_ID],
-            event[mm_constant.ApplicationEvent.OUTPUT_STREAM_URI],
-        )
-
-    @staticmethod
-    def _create_context_for_logging(app_name: str):
-        context = mlrun.get_or_create_ctx(
-            f"{app_name}-logger",
-            upload_artifacts=True,
-            labels={"workflow": "model-monitoring-app-logger"},
-        )
-        return context
-
-    @staticmethod
-    def dict_to_histogram(
-        histogram_dict: mlrun.common.model_monitoring.helpers.FeatureStats,
-    ) -> pd.DataFrame:
-        """
-        Convert histogram dictionary to pandas DataFrame with feature histograms as columns
-
-        :param histogram_dict: Histogram dictionary
-
-        :returns: Histogram dataframe
-        """
-
-        # Create a dictionary with feature histograms as values
-        histograms = {}
-        for feature, stats in histogram_dict.items():
-            if "hist" in stats:
-                # Normalize to probability distribution of each feature
-                histograms[feature] = np.array(stats["hist"][0]) / stats["count"]
-
-        # Convert the dictionary to pandas DataFrame
-        histograms = pd.DataFrame(histograms)
-
-        return histograms
-
-
-class PushToMonitoringWriter(StepToDict):
-    kind = "monitoring_application_stream_pusher"
-
-    def __init__(
-        self,
-        project: Optional[str] = None,
-        writer_application_name: Optional[str] = None,
-        stream_uri: Optional[str] = None,
-        name: Optional[str] = None,
-    ):
-        """
-        Class for pushing application results to the monitoring writer stream.
-
-        :param project:                     Project name.
-        :param writer_application_name:     Writer application name.
-        :param stream_uri:                  Stream URI for pushing results.
-        :param name:                        Name of the PushToMonitoringWriter
-                                            instance default to PushToMonitoringWriter.
-        """
-        self.project = project
-        self.application_name_to_push = writer_application_name
-        self.stream_uri = stream_uri or get_stream_path(
-            project=self.project, function_name=self.application_name_to_push
-        )
-        self.output_stream = None
-        self.name = name or "PushToMonitoringWriter"
-
-    def do(self, event: tuple[list[ModelMonitoringApplicationResult], dict]) -> None:
-        """
-        Push application results to the monitoring writer stream.
-
-        :param event: Monitoring result(s) to push and the original event from the controller.
-        """
-        self._lazy_init()
-        application_results, application_event = event
-        metadata = {
-            mm_constant.WriterEvent.APPLICATION_NAME: application_event[
-                mm_constant.ApplicationEvent.APPLICATION_NAME
-            ],
-            mm_constant.WriterEvent.ENDPOINT_ID: application_event[
-                mm_constant.ApplicationEvent.ENDPOINT_ID
-            ],
-            mm_constant.WriterEvent.START_INFER_TIME: application_event[
-                mm_constant.ApplicationEvent.START_INFER_TIME
-            ],
-            mm_constant.WriterEvent.END_INFER_TIME: application_event[
-                mm_constant.ApplicationEvent.END_INFER_TIME
-            ],
-            mm_constant.WriterEvent.CURRENT_STATS: json.dumps(
-                application_event[mm_constant.ApplicationEvent.CURRENT_STATS]
-            ),
-        }
-        for result in application_results:
-            data = result.to_dict()
-            data.update(metadata)
-            logger.info(f"Pushing data = {data} \n to stream = {self.stream_uri}")
-            self.output_stream.push([data])
-
-    def _lazy_init(self):
-        if self.output_stream is None:
-            self.output_stream = get_stream_pusher(
-                self.stream_uri,
-            )
+# TODO : delete this file in 1.9.0
+from mlrun.model_monitoring.applications import (  # noqa: F401
+    ModelMonitoringApplicationBase,
+    ModelMonitoringApplicationResult,
+)

mlrun/model_monitoring/applications/__init__.py

@@ -11,3 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+#
+
+from .base import ModelMonitoringApplicationBase, ModelMonitoringApplicationBaseV2
+from .context import MonitoringApplicationContext
+from .evidently_base import (
+    _HAS_EVIDENTLY,
+    SUPPORTED_EVIDENTLY_VERSION,
+    EvidentlyModelMonitoringApplicationBase,
+    EvidentlyModelMonitoringApplicationBaseV2,
+)
+from .results import ModelMonitoringApplicationMetric, ModelMonitoringApplicationResult