mlrun 1.8.0rc45__py3-none-any.whl → 1.8.0rc47__py3-none-any.whl

mlrun/model_monitoring/db/_schedules.py

@@ -13,52 +13,38 @@
 # limitations under the License.
 
 import json
+from abc import ABC, abstractmethod
 from contextlib import AbstractContextManager
 from types import TracebackType
 from typing import Final, Optional
 
 import botocore.exceptions
 
-import mlrun.common.schemas
+import mlrun.common.schemas as schemas
 import mlrun.errors
 import mlrun.model_monitoring.helpers
 from mlrun.utils import logger
 
 
-class ModelMonitoringSchedulesFile(AbstractContextManager):
+class ModelMonitoringSchedulesFileBase(AbstractContextManager, ABC):
     DEFAULT_SCHEDULES: Final = {}
     INITIAL_CONTENT = json.dumps(DEFAULT_SCHEDULES)
     ENCODING = "utf-8"
 
-    def __init__(self, project: str, endpoint_id: str) -> None:
-        """
-        Initialize applications monitoring schedules file object.
-        The JSON file stores a dictionary of registered application name as key and Unix timestamp as value.
-        When working with the schedules data, use this class as a context manager to read and write the data.
-
-        :param project:     The project name.
-        :param endpoint_id: The endpoint ID.
-        """
-        # `self._item` is the persistent version of the monitoring schedules.
-        self._item = mlrun.model_monitoring.helpers.get_monitoring_schedules_data(
-            project=project, endpoint_id=endpoint_id
-        )
-        self._path = self._item.url
-        self._fs = self._item.store.filesystem
-        # `self._schedules` is an in-memory copy of the DB for all the applications for
-        # the same model endpoint.
-        self._schedules: dict[str, int] = self.DEFAULT_SCHEDULES.copy()
-        # Does `self._schedules` hold the content of `self._item`?
-        self._open_schedules = False
-
-    @classmethod
-    def from_model_endpoint(
-        cls, model_endpoint: mlrun.common.schemas.ModelEndpoint
-    ) -> "ModelMonitoringSchedulesFile":
-        return cls(
-            project=model_endpoint.metadata.project,
-            endpoint_id=model_endpoint.metadata.uid,
-        )
+    def __init__(self):
+        self._item = self.get_data_item_object()
+        if self._item:
+            self._path = self._item.url
+            self._fs = self._item.store.filesystem
+            # `self._schedules` is an in-memory copy of the DB for all the applications for
+            # the same model endpoint.
+            self._schedules = self.DEFAULT_SCHEDULES.copy()
+            # Does `self._schedules` hold the content of `self._item`?
+            self._open_schedules = False
+
+    @abstractmethod
+    def get_data_item_object(self) -> mlrun.DataItem:
+        pass
 
     def create(self) -> None:
         """Create a schedules file with initial content - an empty dictionary"""
@@ -114,7 +100,7 @@ class ModelMonitoringSchedulesFile(AbstractContextManager):
         self._schedules = self.DEFAULT_SCHEDULES
         self._open_schedules = False
 
-    def __enter__(self) -> "ModelMonitoringSchedulesFile":
+    def __enter__(self) -> "ModelMonitoringSchedulesFileBase":
         self._open()
         return super().__enter__()
 
@@ -132,6 +118,36 @@ class ModelMonitoringSchedulesFile(AbstractContextManager):
                 "Open the schedules file as a context manager first"
             )
 
+
+class ModelMonitoringSchedulesFileEndpoint(ModelMonitoringSchedulesFileBase):
+    def __init__(self, project: str, endpoint_id: str) -> None:
+        """
+        Initialize applications monitoring schedules file object.
+        The JSON file stores a dictionary of registered application name as key and Unix timestamp as value.
+        When working with the schedules data, use this class as a context manager to read and write the data.
+
+        :param project:     The project name.
+        :param endpoint_id: The endpoint ID.
+        """
+        # `self._item` is the persistent version of the monitoring schedules.
+        self._project = project
+        self._endpoint_id = endpoint_id
+        super().__init__()
+
+    def get_data_item_object(self) -> mlrun.DataItem:
+        return mlrun.model_monitoring.helpers.get_monitoring_schedules_endpoint_data(
+            project=self._project, endpoint_id=self._endpoint_id
+        )
+
+    @classmethod
+    def from_model_endpoint(
+        cls, model_endpoint: schemas.ModelEndpoint
+    ) -> "ModelMonitoringSchedulesFileEndpoint":
+        return cls(
+            project=model_endpoint.metadata.project,
+            endpoint_id=model_endpoint.metadata.uid,
+        )
+
     def get_application_time(self, application: str) -> Optional[int]:
         self._check_open_schedules()
         return self._schedules.get(application)
@@ -149,6 +165,68 @@ class ModelMonitoringSchedulesFile(AbstractContextManager):
         return min(self._schedules.values(), default=None)
 
 
+class ModelMonitoringSchedulesFileChief(ModelMonitoringSchedulesFileBase):
+    def __init__(self, project: str) -> None:
+        """
+        Initialize applications monitoring schedules chief file object.
+        The JSON file stores a dictionary of registered model endpoints uid as key and point to a dictionary of
+        "last_request" and "last_analyzed" mapped to two Unix timestamps as values.
+        When working with the schedules data, use this class as a context manager to read and write the data.
+
+        :param project:     The project name.
+        """
+        # `self._item` is the persistent version of the monitoring schedules.
+        self._project = project
+        super().__init__()
+
+    def get_data_item_object(self) -> mlrun.DataItem:
+        return mlrun.model_monitoring.helpers.get_monitoring_schedules_chief_data(
+            project=self._project
+        )
+
+    def get_endpoint_last_request(self, endpoint_uid: str) -> Optional[int]:
+        self._check_open_schedules()
+        if endpoint_uid in self._schedules:
+            return self._schedules[endpoint_uid].get(
+                schemas.model_monitoring.constants.ScheduleChiefFields.LAST_REQUEST
+            )
+        else:
+            return None
+
+    def update_endpoint_timestamps(
+        self, endpoint_uid: str, last_request: int, last_analyzed: int
+    ) -> None:
+        self._check_open_schedules()
+        self._schedules[endpoint_uid] = {
+            schemas.model_monitoring.constants.ScheduleChiefFields.LAST_REQUEST: last_request,
+            schemas.model_monitoring.constants.ScheduleChiefFields.LAST_ANALYZED: last_analyzed,
+        }
+
+    def get_endpoint_last_analyzed(self, endpoint_uid: str) -> Optional[int]:
+        self._check_open_schedules()
+        if endpoint_uid in self._schedules:
+            return self._schedules[endpoint_uid].get(
+                schemas.model_monitoring.constants.ScheduleChiefFields.LAST_ANALYZED
+            )
+        else:
+            return None
+
+    def get_endpoint_list(self) -> set[str]:
+        self._check_open_schedules()
+        return set(self._schedules.keys())
+
+    def get_or_create(self) -> None:
+        try:
+            self._open()
+        except (
+            mlrun.errors.MLRunNotFoundError,
+            # Different errors are raised for S3 or local storage, see ML-8042
+            botocore.exceptions.ClientError,
+            FileNotFoundError,
+        ):
+            self.create()
+
+
 def delete_model_monitoring_schedules_folder(project: str) -> None:
     """Delete the model monitoring schedules folder of the project"""
     folder = mlrun.model_monitoring.helpers._get_monitoring_schedules_folder_path(

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -55,6 +55,9 @@ class TDEngineConnector(TSDBConnector):
 
         self._init_super_tables()
 
+        self._run_directly = (
+            mlrun.mlconf.model_endpoint_monitoring.tdengine.run_directly
+        )
         self._timeout = mlrun.mlconf.model_endpoint_monitoring.tdengine.timeout
         self._retries = mlrun.mlconf.model_endpoint_monitoring.tdengine.retries
 
@@ -74,7 +77,9 @@ class TDEngineConnector(TSDBConnector):
     def _create_connection(self) -> TDEngineConnection:
         """Establish a connection to the TSDB server."""
         logger.debug("Creating a new connection to TDEngine", project=self.project)
-        conn = TDEngineConnection(self._tdengine_connection_profile.dsn())
+        conn = TDEngineConnection(
+            self._tdengine_connection_profile.dsn(), run_directly=self._run_directly
+        )
         conn.prefix_statements = [f"USE {self.database}"]
 
         return conn

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -1090,9 +1090,9 @@ class V3IOTSDBConnector(TSDBConnector):
         Fetch basic metrics from V3IO TSDB and add them to MEP objects.
 
         :param model_endpoint_objects: A list of `ModelEndpoint` objects that will
-                                        be filled with the relevant basic metrics.
+                                       be filled with the relevant basic metrics.
         :param project:                The name of the project.
-        :param run_in_threadpool:      Has no effect.
+        :param run_in_threadpool:      A function that runs another function in a thread pool.
 
         :return: A list of `ModelEndpointMonitoringMetric` objects.
         """
@@ -1104,9 +1104,15 @@ class V3IOTSDBConnector(TSDBConnector):
             uids.append(uid)
             model_endpoint_objects_by_uid[uid] = model_endpoint_object
 
-        error_count_res = self.get_error_count(endpoint_ids=uids, get_raw=True)
-        avg_latency_res = self.get_avg_latency(endpoint_ids=uids, get_raw=True)
-        drift_status_res = self.get_drift_status(endpoint_ids=uids, get_raw=True)
+        error_count_res = await run_in_threadpool(
+            self.get_error_count, endpoint_ids=uids, get_raw=True
+        )
+        avg_latency_res = await run_in_threadpool(
+            self.get_avg_latency, endpoint_ids=uids, get_raw=True
+        )
+        drift_status_res = await run_in_threadpool(
+            self.get_drift_status, endpoint_ids=uids, get_raw=True
+        )
 
         def add_metric(
             metric: str,

mlrun/model_monitoring/helpers.py

@@ -432,57 +432,23 @@ def update_model_endpoint_last_request(
     :param current_request: current request time
     :param db:              DB interface.
     """
-    is_batch_endpoint = (
-        model_endpoint.metadata.endpoint_type == mm_constants.EndpointType.BATCH_EP
-    )
-    if not is_batch_endpoint:
-        logger.info(
-            "Update model endpoint last request time (EP with serving)",
-            project=project,
-            endpoint_id=model_endpoint.metadata.uid,
-            name=model_endpoint.metadata.name,
-            function_name=model_endpoint.spec.function_name,
-            last_request=model_endpoint.status.last_request,
-            current_request=current_request,
-        )
-        db.patch_model_endpoint(
-            project=project,
-            endpoint_id=model_endpoint.metadata.uid,
-            name=model_endpoint.metadata.name,
-            attributes={mm_constants.EventFieldType.LAST_REQUEST: current_request},
-        )
-    else:  # model endpoint without any serving function - close the window "manually"
-        try:
-            time_window = _get_monitoring_time_window_from_controller_run(project, db)
-        except mlrun.errors.MLRunNotFoundError:
-            logger.warn(
-                "Not bumping model endpoint last request time - the monitoring controller isn't deployed yet.\n"
-                "Call `project.enable_model_monitoring()` first."
-            )
-            return
 
-        bumped_last_request = (
-            current_request
-            + time_window
-            + datetime.timedelta(
-                seconds=mlrun.mlconf.model_endpoint_monitoring.parquet_batching_timeout_secs
-            )
-        )
-        logger.info(
-            "Bumping model endpoint last request time (EP without serving)",
-            project=project,
-            endpoint_id=model_endpoint.metadata.uid,
-            last_request=model_endpoint.status.last_request,
-            current_request=current_request.isoformat(),
-            bumped_last_request=bumped_last_request,
-        )
-        db.patch_model_endpoint(
-            project=project,
-            endpoint_id=model_endpoint.metadata.uid,
-            name=model_endpoint.metadata.name,
-            function_name=model_endpoint.spec.function_name,
-            attributes={mm_constants.EventFieldType.LAST_REQUEST: bumped_last_request},
-        )
+    logger.info(
+        "Update model endpoint last request time (EP with serving)",
+        project=project,
+        endpoint_id=model_endpoint.metadata.uid,
+        name=model_endpoint.metadata.name,
+        function_name=model_endpoint.spec.function_name,
+        last_request=model_endpoint.status.last_request,
+        current_request=current_request,
+    )
+    db.patch_model_endpoint(
+        project=project,
+        endpoint_id=model_endpoint.metadata.uid,
+        name=model_endpoint.metadata.name,
+        function_name=model_endpoint.spec.function_name,
+        attributes={mm_constants.EventFieldType.LAST_REQUEST: current_request},
+    )
 
 
 def calculate_inputs_statistics(
@@ -586,16 +552,43 @@ def _get_monitoring_schedules_folder_path(project: str) -> str:
     )
 
 
-def _get_monitoring_schedules_file_path(*, project: str, endpoint_id: str) -> str:
+def _get_monitoring_schedules_file_endpoint_path(
+    *, project: str, endpoint_id: str
+) -> str:
     return os.path.join(
         _get_monitoring_schedules_folder_path(project), f"{endpoint_id}.json"
     )
 
 
-def get_monitoring_schedules_data(*, project: str, endpoint_id: str) -> "DataItem":
+def get_monitoring_schedules_endpoint_data(
+    *, project: str, endpoint_id: str
+) -> "DataItem":
     """
     Get the model monitoring schedules' data item of the project's model endpoint.
     """
     return mlrun.datastore.store_manager.object(
-        _get_monitoring_schedules_file_path(project=project, endpoint_id=endpoint_id)
+        _get_monitoring_schedules_file_endpoint_path(
+            project=project, endpoint_id=endpoint_id
+        )
+    )
+
+
+def get_monitoring_schedules_chief_data(
+    *,
+    project: str,
+) -> "DataItem":
+    """
+    Get the model monitoring schedules' data item of the project's model endpoint.
+    """
+    return mlrun.datastore.store_manager.object(
+        _get_monitoring_schedules_file_chief_path(project=project)
+    )
+
+
+def _get_monitoring_schedules_file_chief_path(
+    *,
+    project: str,
+) -> str:
+    return os.path.join(
+        _get_monitoring_schedules_folder_path(project), f"{project}.json"
     )

mlrun/projects/project.py

@@ -2144,29 +2144,34 @@ class MlrunProject(ModelObj):
         reset_policy: mlrun.common.schemas.alert.ResetPolicy = mlrun.common.schemas.alert.ResetPolicy.AUTO,
     ) -> list[mlrun.alerts.alert.AlertConfig]:
         """
-        :param name:                   The name of the AlertConfig template. It will be combined with mep_id, app-name
-                                       and result name to generate a unique name.
-        :param summary:                Summary of the alert, will be sent in the generated notifications
-        :param endpoints:              The endpoints from which metrics will be retrieved to configure the alerts.
-                                       This `ModelEndpointList` object obtained via the `list_model_endpoints`
-                                       method or created manually using `ModelEndpoint` objects.
-        :param events:                 AlertTrigger event types (EventKind).
-        :param notifications:          List of notifications to invoke once the alert is triggered
-        :param result_names:           Optional. Filters the result names used to create the alert configuration,
-                                       constructed from the app and result_name regex.
-
-                                       For example:
-                                       [`app1.result-*`, `*.result1`]
-                                       will match "mep_uid1.app1.result.result-1" and "mep_uid1.app2.result.result1".
-                                       A specific result_name (not a wildcard) will always create a new alert
-                                       config, regardless of whether the result name exists.
-        :param severity:               Severity of the alert.
-        :param criteria:               When the alert will be triggered based on the
-                                       specified number of events within the defined time period.
-        :param reset_policy:           When to clear the alert. May be "manual" for manual reset of the alert,
-                                       or "auto" if the criteria contains a time period.
-        :returns:                       List of AlertConfig according to endpoints results,
-                                       filtered by result_names.
+        Generate alert configurations based on specified model endpoints and result names, which can be defined
+        explicitly or using regex patterns.
+
+                :param name:                   The name of the AlertConfig template. It will be combined with
+                                               mep id, app name and result name to generate a unique name.
+                :param summary:                Summary of the alert, will be sent in the generated notifications
+                :param endpoints:              The endpoints from which metrics will be retrieved to configure
+                                               the alerts.
+                                               The ModelEndpointList object is obtained via the `list_model_endpoints`
+                                               method or created manually using `ModelEndpoint` objects.
+                :param events:                 AlertTrigger event types (EventKind).
+                :param notifications:          List of notifications to invoke once the alert is triggered
+                :param result_names:           Optional. Filters the result names used to create the alert
+                                               configuration, constructed from the app and result_name regex.
+
+                                               For example:
+                                               [`app1.result-*`, `*.result1`]
+                                               will match "mep_uid1.app1.result.result-1" and
+                                               "mep_uid1.app2.result.result1".
+                                               A specific result_name (not a wildcard) will always create a new alert
+                                               config, regardless of whether the result name exists.
+                :param severity:               Severity of the alert.
+                :param criteria:               The threshold for triggering the alert based on the
+                                               specified number of events within the defined time period.
+                :param reset_policy:           When to clear the alert. Either "manual" for manual reset of the alert,
+                                               or "auto" if the criteria contains a time period.
+                :returns:                      List of AlertConfig according to endpoints results,
+                                               filtered by result_names.
         """
         db = mlrun.db.get_run_db(secrets=self._secrets)
         matching_results = []
@@ -5266,7 +5271,7 @@ class MlrunProject(ModelObj):
             )
 
         # if engine is remote then skip the local file validation
-        if engine and not engine.startswith("remote"):
+        if engine and engine.startswith("remote"):
             return
 
         code_path = self.spec.get_code_path()

mlrun/runtimes/function_reference.py

@@ -36,6 +36,7 @@ class FunctionReference(ModelObj):
         spec=None,
         kind=None,
         name=None,
+        track_models=None,
     ):
         self.url = url
         self.kind = kind
@@ -46,6 +47,7 @@ class FunctionReference(ModelObj):
             spec = spec.to_dict()
         self.spec = spec
         self.code = code
+        self.track_models = track_models
 
         self._function = None
         self._address = None
@@ -130,6 +132,7 @@ class FunctionReference(ModelObj):
         if self.requirements:
             func.with_requirements(self.requirements)
         self._function = func
+        func.spec.track_models = self.track_models
         return func
 
     @property

mlrun/runtimes/nuclio/function.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import asyncio
+import copy
 import json
 import typing
 import warnings
@@ -50,6 +51,19 @@ from mlrun.runtimes.utils import get_item_name, log_std
 from mlrun.utils import get_in, logger, update_in
 from mlrun_pipelines.common.ops import deploy_op
 
+SENSITIVE_PATHS_IN_TRIGGER_CONFIG = {
+    "password",
+    "secret",
+    "attributes/password",
+    "attributes/accesskeyid",
+    "attributes/secretaccesskey",
+    "attributes/cacert",
+    "attributes/accesskey",
+    "attributes/accesscertificate",
+    "attributes/sasl/password",
+    "attributes/sasl/oauth/clientsecret",
+}
+
 
 def validate_nuclio_version_compatibility(*min_versions):
     """
@@ -274,6 +288,37 @@ class RemoteRuntime(KubeResource):
         if self.metadata.tag:
             mlrun.utils.validate_tag_name(self.metadata.tag, "function.metadata.tag")
 
+    def mask_sensitive_data_in_config(self):
+        if not self.spec.config:
+            return {}
+
+        raw_config = copy.deepcopy(self.spec.config)
+
+        for key, value in self.spec.config.items():
+            if key.startswith("spec.triggers"):
+                trigger_name = key.split(".")[-1]
+
+                for path in SENSITIVE_PATHS_IN_TRIGGER_CONFIG:
+                    # Handle nested keys
+                    nested_keys = path.split("/")
+                    target = value
+                    for sub_key in nested_keys[:-1]:
+                        target = target.get(sub_key, {})
+
+                    last_key = nested_keys[-1]
+                    if last_key in target:
+                        sensitive_field = target[last_key]
+                        if sensitive_field.startswith(
+                            mlrun.model.Credentials.secret_reference_prefix
+                        ):
+                            # already masked
+                            continue
+                        target[last_key] = (
+                            f"{mlrun.model.Credentials.secret_reference_prefix}/spec/triggers/{trigger_name}/{path}"
+                        )
+
+        return raw_config
+
     def set_config(self, key, value):
         self.spec.config[key] = value
         return self
@@ -1230,6 +1275,9 @@ class RemoteRuntime(KubeResource):
             if remote_env.get("name") in credentials_env_var_names:
                 new_env.append(remote_env)
 
+        # update nuclio-specific credentials
+        self.mask_sensitive_data_in_config()
+
         self.spec.env = new_env
 
     def _set_as_mock(self, enable):

mlrun/runtimes/nuclio/serving.py

@@ -337,6 +337,17 @@ class ServingRuntime(RemoteRuntime):
         """
         # Applying model monitoring configurations
         self.spec.track_models = enable_tracking
+        if self._spec and self._spec.function_refs:
+            logger.debug(
+                "Set tracking for children references", enable_tracking=enable_tracking
+            )
+            for name in self._spec.function_refs.keys():
+                self._spec.function_refs[name].track_models = enable_tracking
+                # Check if function_refs _function is filled if so update track_models field:
+                if self._spec.function_refs[name]._function:
+                    self._spec.function_refs[
+                        name
+                    ]._function.spec.track_models = enable_tracking
 
         if not 0 < sampling_percentage <= 100:
             raise mlrun.errors.MLRunInvalidArgumentError(
@@ -506,7 +517,11 @@ class ServingRuntime(RemoteRuntime):
         :return function object
         """
         function_reference = FunctionReference(
-            url, image, requirements=requirements, kind=kind or "serving"
+            url,
+            image,
+            requirements=requirements,
+            kind=kind or "serving",
+            track_models=self.spec.track_models,
         )
         self._spec.function_refs.update(function_reference, name)
         func = function_reference.to_function(self.kind)

mlrun/serving/states.py

@@ -363,15 +363,22 @@ class BaseStep(ModelObj):
                             event: {"x": 5} , result_path="y" means the output of the step will be written
                             to event["y"] resulting in {"x": 5, "y": <result>}
         :param model_endpoint_creation_strategy: Strategy for creating or updating the model endpoint:
-            * **overwrite**:
-            1. If model endpoints with the same name exist, delete the `latest` one.
-            2. Create a new model endpoint entry and set it as `latest`.
-            * **inplace** (default):
-            1. If model endpoints with the same name exist, update the `latest` entry.
-            2. Otherwise, create a new entry.
-            * **archive**:
-            1. If model endpoints with the same name exist, preserve them.
-            2. Create a new model endpoint with the same name and set it to `latest`.
+
+                            * **overwrite**:
+
+                            1. If model endpoints with the same name exist, delete the `latest` one.
+                            2. Create a new model endpoint entry and set it as `latest`.
+
+                            * **inplace** (default):
+
+                            1. If model endpoints with the same name exist, update the `latest` entry.
+                            2. Otherwise, create a new entry.
+
+                            * **archive**:
+
+                            1. If model endpoints with the same name exist, preserve them.
+                            2. Create a new model endpoint with the same name and set it to `latest`.
+
         :param class_args:  class init arguments
         """
         if hasattr(self, "steps"):
@@ -810,15 +817,22 @@ class RouterStep(TaskStep):
         :param handler:    class handler to invoke on run/event
         :param function:   function this step should run in
         :param creation_strategy: Strategy for creating or updating the model endpoint:
-            * **overwrite**:
-            1. If model endpoints with the same name exist, delete the `latest` one.
-            2. Create a new model endpoint entry and set it as `latest`.
-            * **inplace** (default):
-            1. If model endpoints with the same name exist, update the `latest` entry.
-            2. Otherwise, create a new entry.
-            * **archive**:
-            1. If model endpoints with the same name exist, preserve them.
-            2. Create a new model endpoint with the same name and set it to `latest`.
+
+                           * **overwrite**:
+
+                           1. If model endpoints with the same name exist, delete the `latest` one.
+                           2. Create a new model endpoint entry and set it as `latest`.
+
+                           * **inplace** (default):
+
+                           1. If model endpoints with the same name exist, update the `latest` entry.
+                           2. Otherwise, create a new entry.
+
+                           * **archive**:
+
+                           1. If model endpoints with the same name exist, preserve them.
+                           2. Create a new model endpoint with the same name and set it to `latest`.
+
         """
 
         if len(self.routes.keys()) >= MAX_MODELS_PER_ROUTER and key not in self.routes:
@@ -1207,15 +1221,22 @@ class FlowStep(BaseStep):
                             event: {"x": 5} , result_path="y" means the output of the step will be written
                             to event["y"] resulting in {"x": 5, "y": <result>}
         :param model_endpoint_creation_strategy: Strategy for creating or updating the model endpoint:
-            * **overwrite**:
-            1. If model endpoints with the same name exist, delete the `latest` one.
-            2. Create a new model endpoint entry and set it as `latest`.
-            * **inplace** (default):
-            1. If model endpoints with the same name exist, update the `latest` entry.
-            2. Otherwise, create a new entry.
-            * **archive**:
-            1. If model endpoints with the same name exist, preserve them.
-            2. Create a new model endpoint with the same name and set it to `latest`.
+
+                            * **overwrite**:
+
+                            1. If model endpoints with the same name exist, delete the `latest` one.
+                            2. Create a new model endpoint entry and set it as `latest`.
+
+                            * **inplace** (default):
+
+                            1. If model endpoints with the same name exist, update the `latest` entry.
+                            2. Otherwise, create a new entry.
+
+                            * **archive**:
+
+                            1. If model endpoints with the same name exist, preserve them.
+                            2. Create a new model endpoint with the same name and set it to `latest`.
+
         :param class_args:  class init arguments
         """