mlrun 1.8.0rc21__py3-none-any.whl → 1.8.0rc24__py3-none-any.whl

mlrun/model_monitoring/stream_processing.py

@@ -29,11 +29,14 @@ import mlrun.model_monitoring.db
 import mlrun.serving.states
 import mlrun.utils
 from mlrun.common.schemas.model_monitoring.constants import (
+    ControllerEvent,
+    ControllerEventKind,
     EndpointType,
     EventFieldType,
     FileTargetKind,
     ProjectSecretKeys,
 )
+from mlrun.datastore import parse_kafka_url
 from mlrun.model_monitoring.db import TSDBConnector
 from mlrun.utils import logger
 
@@ -88,7 +91,9 @@ class EventStreamProcessor:
         self.v3io_framesd = v3io_framesd or mlrun.mlconf.v3io_framesd
         self.v3io_api = v3io_api or mlrun.mlconf.v3io_api
 
-        self.v3io_access_key = v3io_access_key or os.environ.get("V3IO_ACCESS_KEY")
+        self.v3io_access_key = v3io_access_key or mlrun.get_secret_or_env(
+            "V3IO_ACCESS_KEY"
+        )
         self.model_monitoring_access_key = (
             model_monitoring_access_key
             or os.environ.get(ProjectSecretKeys.ACCESS_KEY)
@@ -118,6 +123,7 @@ class EventStreamProcessor:
         self,
         fn: mlrun.runtimes.ServingRuntime,
         tsdb_connector: TSDBConnector,
+        controller_stream_uri: str,
     ) -> None:
         """
         Apply monitoring serving graph to a given serving function. The following serving graph includes about 4 main
@@ -146,6 +152,8 @@ class EventStreamProcessor:
 
         :param fn: A serving function.
         :param tsdb_connector: Time series database connector.
+        :param controller_stream_uri: The controller stream URI. Runs on server api pod so needed to be provided as
+        input
         """
 
         graph = typing.cast(
@@ -209,6 +217,20 @@ class EventStreamProcessor:
             )
 
         apply_map_feature_names()
+        # split the graph between event with error vs valid event
+        graph.add_step(
+            "storey.Filter",
+            "FilterNOP",
+            after="MapFeatureNames",
+            _fn="(event.get('kind', " ") != 'nop_event')",
+        )
+        graph.add_step(
+            "storey.Filter",
+            "ForwardNOP",
+            after="MapFeatureNames",
+            _fn="(event.get('kind', " ") == 'nop_event')",
+        )
+
         tsdb_connector.apply_monitoring_stream_steps(
             graph=graph,
             aggregate_windows=self.aggregate_windows,
@@ -221,7 +243,7 @@ class EventStreamProcessor:
             graph.add_step(
                 "ProcessBeforeParquet",
                 name="ProcessBeforeParquet",
-                after="MapFeatureNames",
+                after="FilterNOP",
                 _fn="(event)",
             )
 
@@ -248,6 +270,44 @@ class EventStreamProcessor:
 
         apply_parquet_target()
 
+        # controller branch
+        def apply_push_controller_stream(stream_uri: str):
+            if stream_uri.startswith("v3io://"):
+                graph.add_step(
+                    ">>",
+                    "controller_stream_v3io",
+                    path=stream_uri,
+                    sharding_func=ControllerEvent.ENDPOINT_ID,
+                    access_key=self.v3io_access_key,
+                    after="ForwardNOP",
+                )
+            elif stream_uri.startswith("kafka://"):
+                topic, brokers = parse_kafka_url(stream_uri)
+                logger.info(
+                    "Controller stream uri for kafka",
+                    stream_uri=stream_uri,
+                    topic=topic,
+                    brokers=brokers,
+                )
+                if isinstance(brokers, list):
+                    path = f"kafka://{brokers[0]}/{topic}"
+                elif isinstance(brokers, str):
+                    path = f"kafka://{brokers}/{topic}"
+                else:
+                    raise mlrun.errors.MLRunInvalidArgumentError(
+                        "Brokers must be a list or str check controller stream uri"
+                    )
+                graph.add_step(
+                    ">>",
+                    "controller_stream_kafka",
+                    path=path,
+                    kafka_brokers=brokers,
+                    _sharding_func="kafka_sharding_func",  # TODO: remove this when storey handle str key
+                    after="ForwardNOP",
+                )
+
+        apply_push_controller_stream(controller_stream_uri)
+
 
 class ProcessBeforeParquet(mlrun.feature_store.steps.MapClass):
     def __init__(self, **kwargs):
@@ -321,6 +381,9 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
 
     def do(self, full_event):
         event = full_event.body
+        if event.get(ControllerEvent.KIND, "") == ControllerEventKind.NOP_EVENT:
+            logger.info("Skipped nop event inside of ProcessEndpointEvent", event=event)
+            return storey.Event(body=[event])
         # Getting model version and function uri from event
         # and use them for retrieving the endpoint_id
         function_uri = full_event.body.get(EventFieldType.FUNCTION_URI)
@@ -589,6 +652,9 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
         return None
 
     def do(self, event: dict):
+        if event.get(ControllerEvent.KIND, "") == ControllerEventKind.NOP_EVENT:
+            logger.info("Skipped nop event inside of MapFeatureNames", event=event)
+            return event
         endpoint_id = event[EventFieldType.ENDPOINT_ID]
 
         feature_values = event[EventFieldType.FEATURES]
@@ -827,3 +893,7 @@ def update_monitoring_feature_set(
         )
 
     monitoring_feature_set.save()
+
+
+def kafka_sharding_func(event):
+    return event.body[ControllerEvent.ENDPOINT_ID].encode("UTF-8")

mlrun/projects/project.py

@@ -29,6 +29,7 @@ import zipfile
 from copy import deepcopy
 from os import environ, makedirs, path
 from typing import Callable, Optional, Union, cast
+from urllib.parse import urlparse
 
 import dotenv
 import git
@@ -1964,17 +1965,17 @@ class MlrunProject(ModelObj):
             ... )
 
         """
+        document_loader_spec = document_loader_spec or DocumentLoaderSpec()
         if not document_loader_spec.download_object and upload:
             raise ValueError(
-                "This document loader expects direct links/URLs and does not support file uploads. "
-                "Either set download_object=True or set upload=False"
+                "The document loader is configured to not support downloads but the upload flag is set to True."
+                "Either set loader.download_object=True or set upload=False"
             )
         doc_artifact = DocumentArtifact(
             key=key,
             original_source=local_path or target_path,
-            document_loader_spec=document_loader_spec
-            if document_loader_spec
-            else DocumentLoaderSpec(),
+            document_loader_spec=document_loader_spec,
+            collections=kwargs.pop("collections", None),
             **kwargs,
         )
         return self.log_artifact(
@@ -3608,9 +3609,12 @@ class MlrunProject(ModelObj):
     def set_model_monitoring_credentials(
         self,
         access_key: Optional[str] = None,
-        stream_path: Optional[str] = None,
-        tsdb_connection: Optional[str] = None,
+        stream_path: Optional[str] = None,  # Deprecated
+        tsdb_connection: Optional[str] = None,  # Deprecated
         replace_creds: bool = False,
+        *,
+        stream_profile_name: Optional[str] = None,
+        tsdb_profile_name: Optional[str] = None,
     ):
         """
         Set the credentials that will be used by the project's model monitoring
@@ -3622,50 +3626,109 @@ class MlrunProject(ModelObj):
                                           * None - will be set from the system configuration.
                                           * v3io - for v3io endpoint store, pass `v3io` and the system will generate the
                                             exact path.
-        :param stream_path:               Path to the model monitoring stream. By default, None. Options:
-
-                                          * None - will be set from the system configuration.
-                                          * v3io - for v3io stream, pass `v3io` and the system will generate the exact
-                                            path.
-                                          * Kafka - for Kafka stream, provide the full connection string without custom
-                                            topic, for example kafka://<some_kafka_broker>:<port>.
-        :param tsdb_connection:           Connection string to the time series database. By default, None.
+        :param stream_path:               (Deprecated) This argument is deprecated. Use ``stream_profile_name`` instead.
+                                          Path to the model monitoring stream. By default, None. Options:
+
+                                          * ``"v3io"`` - for v3io stream, pass ``"v3io"`` and the system will generate
+                                            the exact path.
+                                          * Kafka - for Kafka stream, provide the full connection string without acustom
+                                            topic, for example ``"kafka://<some_kafka_broker>:<port>"``.
+        :param tsdb_connection:           (Deprecated) Connection string to the time series database. By default, None.
                                           Options:
 
-                                          * None - will be set from the system configuration.
-                                          * v3io - for v3io stream, pass `v3io` and the system will generate the exact
-                                            path.
+                                          * v3io - for v3io stream, pass ``"v3io"`` and the system will generate the
+                                            exact path.
                                           * TDEngine - for TDEngine tsdb, provide the full websocket connection URL,
-                                            for example taosws://<username>:<password>@<host>:<port>.
+                                            for example ``"taosws://<username>:<password>@<host>:<port>"``.
         :param replace_creds:             If True, will override the existing credentials.
                                           Please keep in mind that if you already enabled model monitoring on
                                           your project this action can cause data loose and will require redeploying
                                           all model monitoring functions & model monitoring infra
                                           & tracked model server.
+        :param stream_profile_name:       The datastore profile name of the stream to be used in model monitoring.
+                                          The supported profiles are:
+
+                                          * :py:class:`~mlrun.datastore.datastore_profile.DatastoreProfileV3io`
+                                          * :py:class:`~mlrun.datastore.datastore_profile.DatastoreProfileKafkaSource`
+
+                                          You need to register one of them, and pass the profile's name.
+        :param tsdb_profile_name:         The datastore profile name of the time-series database to be used in model
+                                          monitoring. The supported profiles are:
+
+                                          * :py:class:`~mlrun.datastore.datastore_profile.DatastoreProfileV3io`
+                                          * :py:class:`~mlrun.datastore.datastore_profile.TDEngineDatastoreProfile`
+
+                                          You need to register one of them, and pass the profile's name.
         """
         db = mlrun.db.get_run_db(secrets=self._secrets)
-        if tsdb_connection == "v3io":
-            tsdb_profile = mlrun.datastore.datastore_profile.DatastoreProfileV3io(
-                name="mm-infra-tsdb"
+
+        if tsdb_connection:
+            warnings.warn(
+                "The `tsdb_connection` argument is deprecated and will be removed in MLRun version 1.8.0. "
+                "Use `tsdb_profile_name` instead.",
+                FutureWarning,
             )
+            if tsdb_profile_name:
+                raise mlrun.errors.MLRunValueError(
+                    "If you set `tsdb_profile_name`, you must not pass `tsdb_connection`."
+                )
+            if tsdb_connection == "v3io":
+                tsdb_profile = mlrun.datastore.datastore_profile.DatastoreProfileV3io(
+                    name=mm_constants.DefaultProfileName.TSDB
+                )
+            else:
+                parsed_url = urlparse(tsdb_connection)
+                if parsed_url.scheme != "taosws":
+                    raise mlrun.errors.MLRunValueError(
+                        f"Unsupported `tsdb_connection`: '{tsdb_connection}'."
+                    )
+                tsdb_profile = (
+                    mlrun.datastore.datastore_profile.TDEngineDatastoreProfile(
+                        name=mm_constants.DefaultProfileName.TSDB,
+                        user=parsed_url.username,
+                        password=parsed_url.password,
+                        host=parsed_url.hostname,
+                        port=parsed_url.port,
+                    )
+                )
+
             self.register_datastore_profile(tsdb_profile)
             tsdb_profile_name = tsdb_profile.name
-        else:
-            tsdb_profile_name = None
-        if stream_path == "v3io":
-            stream_profile = mlrun.datastore.datastore_profile.DatastoreProfileV3io(
-                name="mm-infra-stream"
+
+        if stream_path:
+            warnings.warn(
+                "The `stream_path` argument is deprecated and will be removed in MLRun version 1.8.0. "
+                "Use `stream_profile_name` instead.",
+                FutureWarning,
             )
+            if stream_profile_name:
+                raise mlrun.errors.MLRunValueError(
+                    "If you set `stream_profile_name`, you must not pass `stream_path`."
+                )
+            if stream_path == "v3io":
+                stream_profile = mlrun.datastore.datastore_profile.DatastoreProfileV3io(
+                    name=mm_constants.DefaultProfileName.STREAM
+                )
+            else:
+                parsed_stream = urlparse(stream_path)
+                if parsed_stream.scheme != "kafka":
+                    raise mlrun.errors.MLRunValueError(
+                        f"Unsupported `stream_path`: '{stream_path}'."
+                    )
+                stream_profile = (
+                    mlrun.datastore.datastore_profile.DatastoreProfileKafkaSource(
+                        name=mm_constants.DefaultProfileName.STREAM,
+                        brokers=[parsed_stream.netloc],
+                        topics=[],
+                    )
+                )
             self.register_datastore_profile(stream_profile)
             stream_profile_name = stream_profile.name
-        else:
-            stream_profile_name = None
+
         db.set_model_monitoring_credentials(
             project=self.name,
             credentials={
                 "access_key": access_key,
-                "stream_path": stream_path,
-                "tsdb_connection": tsdb_connection,
                 "tsdb_profile_name": tsdb_profile_name,
                 "stream_profile_name": stream_profile_name,
             },
@@ -3676,7 +3739,7 @@ class MlrunProject(ModelObj):
                 "Model monitoring credentials were set successfully. "
                 "Please keep in mind that if you already had model monitoring functions "
                 "/ model monitoring infra / tracked model server "
-                "deployed on your project, you will need to redeploy them."
+                "deployed on your project, you will need to redeploy them. "
                 "For redeploying the model monitoring infra, please use `enable_model_monitoring` API "
                 "and set `rebuild_images=True`"
             )

mlrun/runtimes/nuclio/serving.py

@@ -688,7 +688,7 @@ class ServingRuntime(RemoteRuntime):
             "project": self.metadata.project,
             "version": "v2",
             "parameters": self.spec.parameters,
-            "graph": self.spec.graph.to_dict() if self.spec.graph else {},
+            "graph": self.spec.graph.to_dict(strip=True) if self.spec.graph else {},
             "load_mode": self.spec.load_mode,
             "functions": function_name_uri_map,
             "graph_initializer": self.spec.graph_initializer,

mlrun/serving/server.py

@@ -44,6 +44,8 @@ from ..utils import get_caller_globals
 from .states import RootFlowStep, RouterStep, get_function, graph_root_setter
 from .utils import event_id_key, event_path_key
 
+DUMMY_STREAM = "dummy://"
+
 
 class _StreamContext:
     """Handles the stream context for the events stream process. Includes the configuration for the output stream
@@ -72,14 +74,20 @@ class _StreamContext:
                 function_uri, config.default_project
             )
 
-            self.stream_uri = mlrun.model_monitoring.get_stream_path(project=project)
+            stream_args = parameters.get("stream_args", {})
+
+            if log_stream == DUMMY_STREAM:
+                # Dummy stream used for testing, see tests/serving/test_serving.py
+                self.stream_uri = DUMMY_STREAM
+            elif not stream_args.get("mock"):  # if not a mock: `context.is_mock = True`
+                self.stream_uri = mlrun.model_monitoring.get_stream_path(
+                    project=project
+                )
 
             if log_stream:
                 # Update the stream path to the log stream value
                 self.stream_uri = log_stream.format(project=project)
 
-            stream_args = parameters.get("stream_args", {})
-
             self.output_stream = get_stream_pusher(self.stream_uri, **stream_args)
 
 

mlrun/serving/states.py

@@ -31,6 +31,7 @@ import storey.utils
 
 import mlrun
 import mlrun.common.schemas as schemas
+from mlrun.utils import logger
 
 from ..config import config
 from ..datastore import get_stream_pusher
@@ -49,6 +50,8 @@ path_splitter = "/"
 previous_step = "$prev"
 queue_class_names = [">>", "$queue"]
 
+MAX_MODELS_PER_ROUTER = 5000
+
 
 class GraphError(Exception):
     """error in graph topology or configuration"""
@@ -86,8 +89,10 @@ _task_step_fields = [
     "endpoint_type",
 ]
 
-
-MAX_ALLOWED_STEPS = 4500
+_default_fields_to_strip_from_step = [
+    "model_endpoint_creation_strategy",
+    "endpoint_type",
+]
 
 
 def new_remote_endpoint(
@@ -110,6 +115,7 @@ class BaseStep(ModelObj):
     kind = "BaseStep"
     default_shape = "ellipse"
     _dict_fields = ["kind", "comment", "after", "on_error"]
+    _default_fields_to_strip = _default_fields_to_strip_from_step
 
     def __init__(
         self,
@@ -625,6 +631,19 @@ class TaskStep(BaseStep):
                 raise exc
         return event
 
+    def to_dict(
+        self,
+        fields: Optional[list] = None,
+        exclude: Optional[list] = None,
+        strip: bool = False,
+    ) -> dict:
+        self.endpoint_type = (
+            self.endpoint_type.value
+            if isinstance(self.endpoint_type, schemas.EndpointType)
+            else self.endpoint_type
+        )
+        return super().to_dict(fields, exclude, strip)
+
 
 class MonitoringApplicationStep(TaskStep):
     """monitoring application execution step, runs users class code"""
@@ -755,7 +774,7 @@ class RouterStep(TaskStep):
         creation_strategy: schemas.ModelEndpointCreationStrategy = schemas.ModelEndpointCreationStrategy.INPLACE,
         **class_args,
     ):
-        """add child route step or class to the router
+        """add child route step or class to the router, if key exists it will be updated
 
         :param key:        unique name (and route path) for the child step
         :param route:      child step object (Task, ..)
@@ -775,7 +794,13 @@ class RouterStep(TaskStep):
             2. Create a new model endpoint with the same name and set it to `latest`.
 
         """
-
+        if len(self.routes.keys()) >= MAX_MODELS_PER_ROUTER and key not in self.routes:
+            raise mlrun.errors.MLRunModelLimitExceededError(
+                f"Router cannot support more than {MAX_MODELS_PER_ROUTER} model endpoints. "
+                f"To add a new route, edit an existing one by passing the same key."
+            )
+        if key in self.routes:
+            logger.info(f"Model {key} already exists, updating it.")
         if not route and not class_name and not handler:
             raise MLRunInvalidArgumentError("route or class_name must be specified")
         if not route:
@@ -790,10 +815,6 @@ class RouterStep(TaskStep):
             )
         route.function = function or route.function
 
-        if len(self._routes) >= MAX_ALLOWED_STEPS:
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                f"Cannot create the serving graph: the maximum number of steps is {MAX_ALLOWED_STEPS}"
-            )
         route = self._routes.update(key, route)
         route.set_parent(self)
         return route
@@ -806,6 +827,10 @@ class RouterStep(TaskStep):
             del self._routes[key]
 
     def init_object(self, context, namespace, mode="sync", reset=False, **extra_kwargs):
+        if not self.routes:
+            raise mlrun.errors.MLRunRuntimeError(
+                "You have to add models to the router step before initializing it"
+            )
         if not self._is_local_function(context):
             return
 

mlrun/utils/notifications/notification_pusher.py

@@ -412,8 +412,17 @@ class NotificationPusher(_NotificationPusherBase):
         sent_time: typing.Optional[datetime.datetime] = None,
         reason: typing.Optional[str] = None,
     ):
-        if run_state not in runtimes_constants.RunStates.terminal_states():
-            # we want to update the notification status only if the run is in a terminal state for BC
+        # Skip update the notification state if the following conditions are met:
+        # 1. the run is not in a terminal state
+        # 2. the when contains only one state (which is the current state)
+        # Skip updating because currently each notification has only one row in the db, even if it has multiple when.
+        # This means that if the notification is updated to sent for running state for example, it will not send for
+        # The terminal state
+        # TODO: Change this behavior after implementing ML-8723
+        if (
+            run_state not in runtimes_constants.RunStates.terminal_states()
+            and len(notification.when) > 1
+        ):
             logger.debug(
                 "Skip updating notification status - run not in terminal state",
                 run_uid=run_uid,

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "b2650d3be127dcd121931256721bab48ab0351c2",
-  "version": "1.8.0-rc21"
+  "git_commit": "f1df8f0da3910eb0d0603cb299703b00e79947f4",
+  "version": "1.8.0-rc24"
 }

{mlrun-1.8.0rc21.dist-info → mlrun-1.8.0rc24.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: mlrun
-Version: 1.8.0rc21
+Version: 1.8.0rc24
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -225,6 +225,18 @@ Requires-Dist: taos-ws-py==0.3.2; extra == "complete-api"
 Requires-Dist: taoswswrap~=0.3.0; extra == "complete-api"
 Requires-Dist: timelength~=1.1; extra == "complete-api"
 Requires-Dist: uvicorn~=0.32.1; extra == "complete-api"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 <a id="top"></a>
 [![Build Status](https://github.com/mlrun/mlrun/actions/workflows/build.yaml/badge.svg?branch=development)](https://github.com/mlrun/mlrun/actions/workflows/build.yaml?query=branch%3Adevelopment)