mlrun 1.8.0rc29__py3-none-any.whl → 1.8.0rc31__py3-none-any.whl

mlrun/projects/project.py

@@ -29,7 +29,6 @@ import zipfile
 from copy import deepcopy
 from os import environ, makedirs, path
 from typing import Callable, Optional, Union, cast
-from urllib.parse import urlparse
 
 import deprecated
 import dotenv
@@ -71,6 +70,7 @@ from mlrun.datastore.datastore_profile import (
 from mlrun.datastore.vectorstore import VectorStoreCollection
 from mlrun.model_monitoring.helpers import (
     filter_results_by_regex,
+    get_alert_name_from_result_fqn,
     get_result_instance_fqn,
 )
 from mlrun.runtimes.nuclio.function import RemoteRuntime
@@ -1254,8 +1254,14 @@ class MlrunProject(ModelObj):
         mlrun.utils.helpers.validate_builder_source(source, pull_at_runtime, workdir)
 
         self.spec.load_source_on_run = pull_at_runtime
+
+        source_has_changed = source != self.spec.source
         self.spec.source = source or self.spec.source
 
+        # new source should not relay on old workdir
+        if source_has_changed:
+            self.spec.workdir = workdir
+
         if self.spec.source.startswith("git://"):
             source, reference, branch = resolve_git_reference_from_source(source)
             if not branch and not reference:
@@ -1264,7 +1270,6 @@ class MlrunProject(ModelObj):
                     "'git://<url>/org/repo.git#<branch-name or refs/heads/..>'"
                 )
 
-        self.spec.workdir = workdir or self.spec.workdir
         try:
             # reset function objects (to recalculate build attributes)
             self.sync_functions()
@@ -2137,7 +2142,8 @@ class MlrunProject(ModelObj):
         reset_policy: mlrun.common.schemas.alert.ResetPolicy = mlrun.common.schemas.alert.ResetPolicy.AUTO,
     ) -> list[mlrun.alerts.alert.AlertConfig]:
         """
-        :param name:                   AlertConfig name.
+        :param name:                   The name of the AlertConfig template. It will be combined with mep_id, app-name
+                                       and result name to generate a unique name.
         :param summary:                Summary of the alert, will be sent in the generated notifications
         :param endpoints:              The endpoints from which metrics will be retrieved to configure the alerts.
                                        This `ModelEndpointList` object obtained via the `list_model_endpoints`
@@ -2198,10 +2204,11 @@ class MlrunProject(ModelObj):
                 )
         alert_result_names = list(set(specific_result_names + matching_results))
         for result_fqn in alert_result_names:
+            result_fqn_name = get_alert_name_from_result_fqn(result_fqn)
             alerts.append(
                 mlrun.alerts.alert.AlertConfig(
                     project=self.name,
-                    name=name,
+                    name=f"{name}--{result_fqn_name}",
                     summary=summary,
                     severity=severity,
                     entities=alert_constants.EventEntities(
@@ -2217,6 +2224,12 @@ class MlrunProject(ModelObj):
                     reset_policy=reset_policy,
                 )
             )
+        if not alerts:
+            warnings.warn(
+                "No alert config has been created. "
+                "Try specifying a result name explicitly or verifying that results are available"
+            )
+
         return alerts
 
     def set_model_monitoring_function(
@@ -3456,12 +3469,13 @@ class MlrunProject(ModelObj):
 
         arguments = arguments or {}
         need_repo = self.spec._need_repo()
-        if self.spec.repo and self.spec.repo.is_dirty():
-            msg = "You seem to have uncommitted git changes, use .push()"
-            if dirty or not need_repo:
-                logger.warning("WARNING!, " + msg)
-            else:
-                raise ProjectError(msg + " or dirty=True")
+        if not dirty:
+            if self.spec.repo and self.spec.repo.is_dirty():
+                msg = "You seem to have uncommitted git changes, use .push()"
+                if not need_repo:
+                    logger.warning("WARNING!, " + msg)
+                else:
+                    raise ProjectError(msg + " or dirty=True")
 
         if need_repo and self.spec.repo and not self.spec.source:
             raise ProjectError(
@@ -3659,50 +3673,77 @@ class MlrunProject(ModelObj):
 
     def set_model_monitoring_credentials(
         self,
-        access_key: Optional[str] = None,
-        stream_path: Optional[str] = None,  # Deprecated
-        tsdb_connection: Optional[str] = None,  # Deprecated
-        replace_creds: bool = False,
         *,
-        stream_profile_name: Optional[str] = None,
-        tsdb_profile_name: Optional[str] = None,
-    ):
+        tsdb_profile_name: str,
+        stream_profile_name: str,
+        replace_creds: bool = False,
+    ) -> None:
         """
-        Set the credentials that will be used by the project's model monitoring
-        infrastructure functions. Important to note that you have to set the credentials before deploying any
-        model monitoring or serving function.
-
-        :param access_key:                Model monitoring access key for managing user permissions.
-
-                                          * None - will be set from the system configuration.
-                                          * v3io - for v3io endpoint store, pass `v3io` and the system will generate the
-                                            exact path.
-        :param stream_path:               (Deprecated) This argument is deprecated. Use ``stream_profile_name`` instead.
-                                          Path to the model monitoring stream. By default, None. Options:
-
-                                          * ``"v3io"`` - for v3io stream, pass ``"v3io"`` and the system will generate
-                                            the exact path.
-                                          * Kafka - for Kafka stream, provide the full connection string without acustom
-                                            topic, for example ``"kafka://<some_kafka_broker>:<port>"``.
-        :param tsdb_connection:           (Deprecated) Connection string to the time series database. By default, None.
-                                          Options:
-
-                                          * v3io - for v3io stream, pass ``"v3io"`` and the system will generate the
-                                            exact path.
-                                          * TDEngine - for TDEngine tsdb, provide the full websocket connection URL,
-                                            for example ``"taosws://<username>:<password>@<host>:<port>"``.
-        :param replace_creds:             If True, will override the existing credentials.
-                                          Please keep in mind that if you already enabled model monitoring on
-                                          your project this action can cause data loose and will require redeploying
-                                          all model monitoring functions & model monitoring infra
-                                          & tracked model server.
-        :param stream_profile_name:       The datastore profile name of the stream to be used in model monitoring.
-                                          The supported profiles are:
+        Set the credentials that will be used by the project's model monitoring infrastructure functions.
+        Please note that you have to set the credentials before deploying any model monitoring application
+        or a tracked serving function.
 
-                                          * :py:class:`~mlrun.datastore.datastore_profile.DatastoreProfileV3io`
-                                          * :py:class:`~mlrun.datastore.datastore_profile.DatastoreProfileKafkaSource`
+        For example, the full flow for enabling model monitoring infrastructure with **TDEngine** and **Kafka**, is:
+
+        .. code-block:: python
+
+            import mlrun
+            from mlrun.datastore.datastore_profile import (
+                DatastoreProfileKafkaSource,
+                TDEngineDatastoreProfile,
+            )
+
+            project = mlrun.get_or_create_project("mm-infra-setup")
+
+            # Create and register TSDB profile
+            tsdb_profile = TDEngineDatastoreProfile(
+                name="my-tdengine",
+                host="<tdengine-server-ip-address>",
+                port=6041,
+                user="username",
+                password="<tdengine-password>",
+            )
+            project.register_datastore_profile(tsdb_profile)
+
+            # Create and register stream profile
+            stream_profile = DatastoreProfileKafkaSource(
+                name="my-kafka",
+                brokers=["<kafka-broker-ip-address>:9094"],
+                topics=[],  # Keep the topics list empty
+                ## SASL is supported
+                # sasl_user="user1",
+                # sasl_pass="<kafka-sasl-password>",
+            )
+            project.register_datastore_profile(stream_profile)
+
+            # Set model monitoring credentials and enable the infrastructure
+            project.set_model_monitoring_credentials(
+                tsdb_profile_name=tsdb_profile.name,
+                stream_profile_name=stream_profile.name,
+            )
+            project.enable_model_monitoring()
+
+        Note that you will need to change the profiles if you want to use **V3IO** TSDB and stream:
+
+        .. code-block:: python
+
+            from mlrun.datastore.datastore_profile import DatastoreProfileV3io
+
+            # Create and register TSDB profile
+            tsdb_profile = DatastoreProfileV3io(
+                name="my-v3io-tsdb",
+            )
+            project.register_datastore_profile(tsdb_profile)
+
+            # Create and register stream profile
+            stream_profile = DatastoreProfileV3io(
+                name="my-v3io-stream",
+                v3io_access_key=mlrun.mlconf.get_v3io_access_key(),
+            )
+            project.register_datastore_profile(stream_profile)
+
+        In the V3IO datastore, you must provide an explicit access key to the stream, but not to the TSDB.
 
-                                          You need to register one of them, and pass the profile's name.
         :param tsdb_profile_name:         The datastore profile name of the time-series database to be used in model
                                           monitoring. The supported profiles are:
 
@@ -3710,76 +3751,24 @@ class MlrunProject(ModelObj):
                                           * :py:class:`~mlrun.datastore.datastore_profile.TDEngineDatastoreProfile`
 
                                           You need to register one of them, and pass the profile's name.
-        """
-        db = mlrun.db.get_run_db(secrets=self._secrets)
-
-        if tsdb_connection:
-            warnings.warn(
-                "The `tsdb_connection` argument is deprecated and will be removed in MLRun version 1.8.0. "
-                "Use `tsdb_profile_name` instead.",
-                FutureWarning,
-            )
-            if tsdb_profile_name:
-                raise mlrun.errors.MLRunValueError(
-                    "If you set `tsdb_profile_name`, you must not pass `tsdb_connection`."
-                )
-            if tsdb_connection == "v3io":
-                tsdb_profile = mlrun.datastore.datastore_profile.DatastoreProfileV3io(
-                    name=mm_constants.DefaultProfileName.TSDB
-                )
-            else:
-                parsed_url = urlparse(tsdb_connection)
-                if parsed_url.scheme != "taosws":
-                    raise mlrun.errors.MLRunValueError(
-                        f"Unsupported `tsdb_connection`: '{tsdb_connection}'."
-                    )
-                tsdb_profile = (
-                    mlrun.datastore.datastore_profile.TDEngineDatastoreProfile(
-                        name=mm_constants.DefaultProfileName.TSDB,
-                        user=parsed_url.username,
-                        password=parsed_url.password,
-                        host=parsed_url.hostname,
-                        port=parsed_url.port,
-                    )
-                )
+        :param stream_profile_name:       The datastore profile name of the stream to be used in model monitoring.
+                                          The supported profiles are:
 
-            self.register_datastore_profile(tsdb_profile)
-            tsdb_profile_name = tsdb_profile.name
+                                          * :py:class:`~mlrun.datastore.datastore_profile.DatastoreProfileV3io`
+                                          * :py:class:`~mlrun.datastore.datastore_profile.DatastoreProfileKafkaSource`
 
-        if stream_path:
-            warnings.warn(
-                "The `stream_path` argument is deprecated and will be removed in MLRun version 1.8.0. "
-                "Use `stream_profile_name` instead.",
-                FutureWarning,
-            )
-            if stream_profile_name:
-                raise mlrun.errors.MLRunValueError(
-                    "If you set `stream_profile_name`, you must not pass `stream_path`."
-                )
-            if stream_path == "v3io":
-                stream_profile = mlrun.datastore.datastore_profile.DatastoreProfileV3io(
-                    name=mm_constants.DefaultProfileName.STREAM
-                )
-            else:
-                parsed_stream = urlparse(stream_path)
-                if parsed_stream.scheme != "kafka":
-                    raise mlrun.errors.MLRunValueError(
-                        f"Unsupported `stream_path`: '{stream_path}'."
-                    )
-                stream_profile = (
-                    mlrun.datastore.datastore_profile.DatastoreProfileKafkaSource(
-                        name=mm_constants.DefaultProfileName.STREAM,
-                        brokers=[parsed_stream.netloc],
-                        topics=[],
-                    )
-                )
-            self.register_datastore_profile(stream_profile)
-            stream_profile_name = stream_profile.name
+                                          You need to register one of them, and pass the profile's name.
+        :param replace_creds:             If ``True`` - override the existing credentials.
+                                          Please keep in mind that if you have already enabled model monitoring
+                                          on your project, replacing the credentials can cause data loss, and will
+                                          require redeploying all the model monitoring functions, model monitoring
+                                          infrastructure, and tracked model servers.
+        """
+        db = mlrun.db.get_run_db(secrets=self._secrets)
 
         db.set_model_monitoring_credentials(
             project=self.name,
             credentials={
-                "access_key": access_key,
                 "tsdb_profile_name": tsdb_profile_name,
                 "stream_profile_name": stream_profile_name,
             },
@@ -4120,7 +4109,7 @@ class MlrunProject(ModelObj):
 
         :param image: target image name/path. If not specified the project's existing `default_image` name will be
                         used. If not set, the `mlconf.default_project_image_name` value will be used
-        :param set_as_default: set `image` to be the project's default image (default False)
+        :param set_as_default: set `image` to be the project's default image (default True)
         :param with_mlrun:      add the current mlrun package to the container build
         :param base_image:      base image name/path (commands and source code will be added to it) defaults to
                                 mlrun.mlconf.default_base_image

mlrun/run.py

@@ -911,7 +911,7 @@ def _run_pipeline(
 
 def retry_pipeline(
     run_id: str,
-    project: Optional[str] = None,
+    project: str,
     namespace: Optional[str] = None,
 ) -> str:
     """Retry a pipeline run.
@@ -920,7 +920,7 @@ def retry_pipeline(
     retryable state, a new run is created as a clone of the original run.
 
     :param run_id: ID of the pipeline run to retry.
-    :param project: Optional; name of the project associated with the pipeline run.
+    :param project: name of the project associated with the pipeline run.
     :param namespace: Optional; Kubernetes namespace to use if not the default.
 
     :returns: ID of the retried pipeline run or the ID of a cloned run if the original run is not retryable.

mlrun/runtimes/nuclio/function.py

@@ -369,8 +369,9 @@ class RemoteRuntime(KubeResource):
                 )
         """
         self.spec.build.source = source
-        # update handler in function_handler
-        self.spec.function_handler = handler
+        # update handler in function_handler if needed
+        if handler:
+            self.spec.function_handler = handler
         if workdir:
             self.spec.workdir = workdir
         if runtime:
@@ -1016,6 +1017,7 @@ class RemoteRuntime(KubeResource):
     ):
         """
         Add a sidecar container to the function pod
+
         :param name:    Sidecar container name.
         :param image:   Sidecar container image.
         :param ports:   Sidecar container ports to expose. Can be a single port or a list of ports.

mlrun/serving/routers.py

@@ -596,7 +596,6 @@ class VotingEnsemble(ParallelRun):
         self.vote_type = vote_type
         self.vote_flag = True if self.vote_type is not None else False
         self.weights = weights
-        self.version = kwargs.get("version", "v1")
         self.log_router = True
         self.prediction_col_name = prediction_col_name or "prediction"
         self.format_response_with_col_name_flag = format_response_with_col_name_flag
@@ -927,14 +926,14 @@ class VotingEnsemble(ParallelRun):
                     "model_name": self.name,
                     "outputs": votes,
                 }
-                if self.version:
-                    response_body["model_version"] = self.version
+                if self.model_endpoint_uid:
+                    response_body["model_endpoint_uid"] = self.model_endpoint_uid
                 response.body = response_body
             elif name == self.name and event.method == "GET" and not subpath:
                 response = copy.copy(event)
                 response_body = {
                     "name": self.name,
-                    "version": self.version or "",
+                    "model_endpoint_uid": self.model_endpoint_uid or "",
                     "inputs": [],
                     "outputs": [],
                 }

mlrun/serving/server.py

@@ -65,7 +65,7 @@ class _StreamContext:
         self.hostname = socket.gethostname()
         self.function_uri = function_uri
         self.output_stream = None
-        self.stream_uri = None
+        stream_uri = None
         log_stream = parameters.get(FileTargetKind.LOG_STREAM, "")
 
         if (enabled or log_stream) and function_uri:
@@ -78,17 +78,19 @@ class _StreamContext:
 
             if log_stream == DUMMY_STREAM:
                 # Dummy stream used for testing, see tests/serving/test_serving.py
-                self.stream_uri = DUMMY_STREAM
+                stream_uri = DUMMY_STREAM
             elif not stream_args.get("mock"):  # if not a mock: `context.is_mock = True`
-                self.stream_uri = mlrun.model_monitoring.get_stream_path(
-                    project=project
-                )
+                stream_uri = mlrun.model_monitoring.get_stream_path(project=project)
 
             if log_stream:
                 # Update the stream path to the log stream value
-                self.stream_uri = log_stream.format(project=project)
-
-            self.output_stream = get_stream_pusher(self.stream_uri, **stream_args)
+                stream_uri = log_stream.format(project=project)
+                self.output_stream = get_stream_pusher(stream_uri, **stream_args)
+            else:
+                # Get the output stream from the profile
+                self.output_stream = mlrun.model_monitoring.helpers.get_output_stream(
+                    project=project, mock=stream_args.get("mock", False)
+                )
 
 
 class GraphServer(ModelObj):

mlrun/serving/states.py

@@ -652,6 +652,14 @@ class TaskStep(BaseStep):
             if isinstance(self.endpoint_type, schemas.EndpointType)
             else self.endpoint_type
         )
+        self.model_endpoint_creation_strategy = (
+            self.model_endpoint_creation_strategy.value
+            if isinstance(
+                self.model_endpoint_creation_strategy,
+                schemas.ModelEndpointCreationStrategy,
+            )
+            else self.model_endpoint_creation_strategy
+        )
         return super().to_dict(fields, exclude, strip)
 
 
@@ -755,9 +763,11 @@ class RouterStep(TaskStep):
         self._routes: ObjectDict = None
         self.routes = routes
         self.endpoint_type = schemas.EndpointType.ROUTER
+        if isinstance(class_name, type):
+            class_name = class_name.__name__
         self.model_endpoint_creation_strategy = (
             schemas.ModelEndpointCreationStrategy.INPLACE
-            if class_name and "serving.VotingEnsemble" in class_name
+            if class_name and "VotingEnsemble" in class_name
             else schemas.ModelEndpointCreationStrategy.SKIP
         )
 
@@ -802,8 +812,8 @@ class RouterStep(TaskStep):
             * **archive**:
             1. If model endpoints with the same name exist, preserve them.
             2. Create a new model endpoint with the same name and set it to `latest`.
-
         """
+
         if len(self.routes.keys()) >= MAX_MODELS_PER_ROUTER and key not in self.routes:
             raise mlrun.errors.MLRunModelLimitExceededError(
                 f"Router cannot support more than {MAX_MODELS_PER_ROUTER} model endpoints. "

mlrun/serving/v2_serving.py

@@ -95,9 +95,6 @@ class V2ModelServer(StepToDict):
         :param kwargs:     extra arguments (can be accessed using self.get_param(key))
         """
         self.name = name
-        self.version = ""
-        if name and ":" in name:
-            self.version = name.split(":", 1)[-1]
         self.context = context
         self.ready = False
         self.error = ""
@@ -118,6 +115,7 @@ class V2ModelServer(StepToDict):
         self.shard_by_endpoint = shard_by_endpoint
         self._model_logger = None
         self.initialized = False
+        self.output_schema = []
 
     def _load_and_update_state(self):
         try:
@@ -178,17 +176,15 @@ class V2ModelServer(StepToDict):
                     "Model endpoint creation task name not provided",
                 )
             try:
-                self.model_endpoint_uid = (
-                    mlrun.get_run_db()
-                    .get_model_endpoint(
-                        project=server.project,
-                        name=self.name,
-                        function_name=server.function_name,
-                        function_tag=server.function_tag or "latest",
-                        tsdb_metrics=False,
-                    )
-                    .metadata.uid
+                model_endpoint = mlrun.get_run_db().get_model_endpoint(
+                    project=server.project,
+                    name=self.name,
+                    function_name=server.function_name,
+                    function_tag=server.function_tag or "latest",
+                    tsdb_metrics=False,
                 )
+                self.model_endpoint_uid = model_endpoint.metadata.uid
+                self.output_schema = model_endpoint.spec.label_names
             except mlrun.errors.MLRunNotFoundError:
                 logger.info(
                     "Model endpoint not found for this step; monitoring for this model will not be performed",
@@ -325,8 +321,8 @@ class V2ModelServer(StepToDict):
                 "outputs": outputs,
                 "timestamp": start.isoformat(sep=" ", timespec="microseconds"),
             }
-            if self.version:
-                response["model_version"] = self.version
+            if self.model_endpoint_uid:
+                response["model_endpoint_uid"] = self.model_endpoint_uid
         elif op == "ready" and event.method == "GET":
             # get model health operation
             setattr(event, "terminated", True)
@@ -352,7 +348,7 @@ class V2ModelServer(StepToDict):
             setattr(event, "terminated", True)
             event_body = {
                 "name": self.name.split(":")[0],
-                "version": self.version or "",
+                "model_endpoint_uid": self.model_endpoint_uid or "",
                 "inputs": [],
                 "outputs": [],
             }
@@ -386,8 +382,8 @@ class V2ModelServer(StepToDict):
                 "model_name": self.name,
                 "outputs": outputs,
             }
-            if self.version:
-                response["model_version"] = self.version
+            if self.model_endpoint_uid:
+                response["model_endpoint_uid"] = self.model_endpoint_uid
 
         elif hasattr(self, "op_" + op):
             # custom operation (child methods starting with "op_")
@@ -510,7 +506,6 @@ class _ModelLogPusher:
         self.verbose = context.verbose
         self.hostname = context.stream.hostname
         self.function_uri = context.stream.function_uri
-        self.stream_path = context.stream.stream_uri
         self.sampling_percentage = float(context.get_param("sampling_percentage", 100))
         self.output_stream = output_stream or context.stream.output_stream
         self._worker = context.worker_id
@@ -520,7 +515,6 @@ class _ModelLogPusher:
             "class": self.model.__class__.__name__,
             "worker": self._worker,
             "model": self.model.name,
-            "version": self.model.version,
             "host": self.hostname,
             "function_uri": self.function_uri,
             "endpoint_id": self.model.model_endpoint_uid,
@@ -571,6 +565,17 @@ class _ModelLogPusher:
                     resp["outputs"] = [
                         resp["outputs"][i] for i in sampled_requests_indices
                     ]
+                if self.model.output_schema and len(self.model.output_schema) != len(
+                    resp["outputs"][0]
+                ):
+                    logger.info(
+                        "The number of outputs returned by the model does not match the number of outputs "
+                        "specified in the model endpoint.",
+                        model_endpoint=self.model.name,
+                        model_endpoint_id=self.model.model_endpoint_uid,
+                        output_len=len(resp["outputs"][0]),
+                        schema_len=len(self.model.output_schema),
+                    )
 
             data = self.base_data()
             data["request"] = request