mlrun 1.7.0rc36__py3-none-any.whl → 1.7.0rc38__py3-none-any.whl

mlrun/model_monitoring/stream_processing.py

@@ -169,11 +169,40 @@ class EventStreamProcessor:
             mlrun.serving.states.RootFlowStep,
             fn.set_topology(mlrun.serving.states.StepKinds.flow),
         )
+        graph.add_step(
+            "ExtractEndpointID",
+            "extract_endpoint",
+            full_event=True,
+        )
+
+        # split the graph between event with error vs valid event
+        graph.add_step(
+            "storey.Filter",
+            "FilterError",
+            after="extract_endpoint",
+            _fn="(event.get('error') is None)",
+        )
+
+        graph.add_step(
+            "storey.Filter",
+            "ForwardError",
+            after="extract_endpoint",
+            _fn="(event.get('error') is not None)",
+        )
+
+        tsdb_connector = mlrun.model_monitoring.get_tsdb_connector(
+            project=self.project, secret_provider=secret_provider
+        )
+
+        tsdb_connector.handle_model_error(
+            graph,
+        )
 
         # Process endpoint event: splitting into sub-events and validate event data
         def apply_process_endpoint_event():
             graph.add_step(
                 "ProcessEndpointEvent",
+                after="FilterError",
                 full_event=True,
                 project=self.project,
             )
@@ -295,9 +324,6 @@ class EventStreamProcessor:
 
         apply_storey_sample_window()
 
-        tsdb_connector = mlrun.model_monitoring.get_tsdb_connector(
-            project=self.project, secret_provider=secret_provider
-        )
         tsdb_connector.apply_monitoring_stream_steps(graph=graph)
 
         # Parquet branch
@@ -386,6 +412,38 @@ class ProcessBeforeEndpointUpdate(mlrun.feature_store.steps.MapClass):
         return e
 
 
+class ExtractEndpointID(mlrun.feature_store.steps.MapClass):
+    def __init__(self, **kwargs) -> None:
+        """
+        Generate the model endpoint ID based on the event parameters and attach it to the event.
+        """
+        super().__init__(**kwargs)
+
+    def do(self, full_event) -> typing.Union[storey.Event, None]:
+        # Getting model version and function uri from event
+        # and use them for retrieving the endpoint_id
+        function_uri = full_event.body.get(EventFieldType.FUNCTION_URI)
+        if not is_not_none(function_uri, [EventFieldType.FUNCTION_URI]):
+            return None
+
+        model = full_event.body.get(EventFieldType.MODEL)
+        if not is_not_none(model, [EventFieldType.MODEL]):
+            return None
+
+        version = full_event.body.get(EventFieldType.VERSION)
+        versioned_model = f"{model}:{version}" if version else f"{model}:latest"
+
+        endpoint_id = mlrun.common.model_monitoring.create_model_endpoint_uid(
+            function_uri=function_uri,
+            versioned_model=versioned_model,
+        )
+
+        endpoint_id = str(endpoint_id)
+        full_event.body[EventFieldType.ENDPOINT_ID] = endpoint_id
+        full_event.body[EventFieldType.VERSIONED_MODEL] = versioned_model
+        return full_event
+
+
 class ProcessBeforeParquet(mlrun.feature_store.steps.MapClass):
     def __init__(self, **kwargs):
         """
@@ -459,28 +517,9 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
     def do(self, full_event):
         event = full_event.body
 
-        # Getting model version and function uri from event
-        # and use them for retrieving the endpoint_id
-        function_uri = event.get(EventFieldType.FUNCTION_URI)
-        if not is_not_none(function_uri, [EventFieldType.FUNCTION_URI]):
-            return None
-
-        model = event.get(EventFieldType.MODEL)
-        if not is_not_none(model, [EventFieldType.MODEL]):
-            return None
-
-        version = event.get(EventFieldType.VERSION)
-        versioned_model = f"{model}:{version}" if version else f"{model}:latest"
-
-        endpoint_id = mlrun.common.model_monitoring.create_model_endpoint_uid(
-            function_uri=function_uri,
-            versioned_model=versioned_model,
-        )
-
-        endpoint_id = str(endpoint_id)
-
-        event[EventFieldType.VERSIONED_MODEL] = versioned_model
-        event[EventFieldType.ENDPOINT_ID] = endpoint_id
+        versioned_model = event[EventFieldType.VERSIONED_MODEL]
+        endpoint_id = event[EventFieldType.ENDPOINT_ID]
+        function_uri = event[EventFieldType.FUNCTION_URI]
 
         # In case this process fails, resume state from existing record
         self.resume_state(endpoint_id)
@@ -598,6 +637,9 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
                     EventFieldType.PREDICTION: prediction,
                     EventFieldType.FIRST_REQUEST: self.first_request[endpoint_id],
                     EventFieldType.LAST_REQUEST: self.last_request[endpoint_id],
+                    EventFieldType.LAST_REQUEST_TIMESTAMP: mlrun.utils.enrich_datetime_with_tz_info(
+                        self.last_request[endpoint_id]
+                    ).timestamp(),
                     EventFieldType.ERROR_COUNT: self.error_count[endpoint_id],
                     EventFieldType.LABELS: event.get(EventFieldType.LABELS, {}),
                     EventFieldType.METRICS: event.get(EventFieldType.METRICS, {}),

mlrun/model_monitoring/writer.py

@@ -130,7 +130,6 @@ class ModelMonitoringWriter(StepToDict):
         project_name: str,
         result_kind: int,
     ) -> None:
-        logger.info("Sending an event")
         entity = mlrun.common.schemas.alert.EventEntities(
             kind=alert_objects.EventEntityKind.MODEL_ENDPOINT_RESULT,
             project=project_name,
@@ -146,7 +145,9 @@ class ModelMonitoringWriter(StepToDict):
             entity=entity,
             value_dict=event_value,
         )
+        logger.info("Sending a drift event")
         mlrun.get_run_db().generate_event(event_kind, event_data)
+        logger.info("Drift event sent successfully")
 
     @staticmethod
     def _generate_alert_event_kind(
@@ -261,3 +262,5 @@ class ModelMonitoringWriter(StepToDict):
                 endpoint_id=endpoint_id,
                 attributes=json.loads(event[ResultData.RESULT_EXTRA_DATA]),
             )
+
+        logger.info("Model monitoring writer finished handling event")

mlrun/projects/operations.py

@@ -187,6 +187,10 @@ def run_function(
     task.spec.verbose = task.spec.verbose or verbose
 
     if engine == "kfp":
+        if schedule:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Scheduling jobs is not supported when running a workflow with the kfp engine."
+            )
         return function.as_step(
             name=name, runspec=task, workdir=workdir, outputs=outputs, labels=labels
         )

mlrun/projects/project.py

@@ -600,6 +600,10 @@ def _run_project_setup(
     if hasattr(mod, "setup"):
         try:
             project = getattr(mod, "setup")(project)
+            if not project or not isinstance(project, mlrun.projects.MlrunProject):
+                raise ValueError(
+                    "MLRun project_setup:setup() must return a project object"
+                )
         except Exception as exc:
             logger.error(
                 "Failed to run project_setup script",
@@ -610,7 +614,9 @@ def _run_project_setup(
         if save:
             project.save()
     else:
-        logger.warn("skipping setup, setup() handler was not found in project_setup.py")
+        logger.warn(
+            f"skipping setup, setup() handler was not found in {path.basename(setup_file_path)}"
+        )
     return project
 
 
@@ -2967,6 +2973,7 @@ class MlrunProject(ModelObj):
         source: str = None,
         cleanup_ttl: int = None,
         notifications: list[mlrun.model.Notification] = None,
+        send_start_notification: bool = True,
     ) -> _PipelineRunStatus:
         """Run a workflow using kubeflow pipelines
 
@@ -3003,6 +3010,8 @@ class MlrunProject(ModelObj):
                           workflow and all its resources are deleted)
         :param notifications:
                           List of notifications to send for workflow completion
+        :param send_start_notification:
+                          Send a notification when the workflow starts
 
         :returns: ~py:class:`~mlrun.projects.pipelines._PipelineRunStatus` instance
         """
@@ -3080,6 +3089,7 @@ class MlrunProject(ModelObj):
             namespace=namespace,
             source=source,
             notifications=notifications,
+            send_start_notification=send_start_notification,
         )
         # run is None when scheduling
         if run and run.state == mlrun_pipelines.common.models.RunStatuses.failed:

mlrun/runtimes/__init__.py

@@ -30,6 +30,8 @@ __all__ = [
     "MpiRuntimeV1",
 ]
 
+import typing
+
 from mlrun.runtimes.utils import resolve_spark_operator_version
 
 from ..common.runtimes.constants import MPIJobCRDVersions
@@ -181,7 +183,7 @@ class RuntimeKinds:
         ]
 
     @staticmethod
-    def is_log_collectable_runtime(kind: str):
+    def is_log_collectable_runtime(kind: typing.Optional[str]):
         """
         whether log collector can collect logs for that runtime
         :param kind: kind name
@@ -192,13 +194,18 @@ class RuntimeKinds:
         if RuntimeKinds.is_local_runtime(kind):
             return False
 
-        if kind not in [
-            # dask implementation is different than other runtimes, because few runs can be run against the same runtime
-            # resource, so collecting logs on that runtime resource won't be correct, the way we collect logs for dask
-            # is by using `log_std` on client side after we execute the code against the cluster, as submitting the
-            # run with the dask client will return the run stdout. for more information head to `DaskCluster._run`
-            RuntimeKinds.dask
-        ]:
+        if (
+            kind
+            not in [
+                # dask implementation is different from other runtimes, because few runs can be run against the same
+                # runtime resource, so collecting logs on that runtime resource won't be correct, the way we collect
+                # logs for dask is by using `log_std` on client side after we execute the code against the cluster,
+                # as submitting the run with the dask client will return the run stdout.
+                # For more information head to `DaskCluster._run`.
+                RuntimeKinds.dask
+            ]
+            + RuntimeKinds.nuclio_runtimes()
+        ):
             return True
 
         return False

mlrun/runtimes/base.py

@@ -929,3 +929,6 @@ class BaseRuntime(ModelObj):
                         if "default" in p:
                             line += f", default={p['default']}"
                         print("    " + line)
+
+    def skip_image_enrichment(self):
+        return False

mlrun/runtimes/nuclio/application/application.py

@@ -18,6 +18,7 @@ import nuclio
 
 import mlrun.common.schemas as schemas
 import mlrun.errors
+import mlrun.run
 from mlrun.common.runtimes.constants import NuclioIngressAddTemplatedIngressModes
 from mlrun.runtimes import RemoteRuntime
 from mlrun.runtimes.nuclio import min_nuclio_versions
@@ -121,6 +122,11 @@ class ApplicationSpec(NuclioSpec):
             state_thresholds=state_thresholds,
             disable_default_http_trigger=disable_default_http_trigger,
         )
+
+        # Override default min/max replicas (don't assume application is stateless)
+        self.min_replicas = min_replicas or 1
+        self.max_replicas = max_replicas or 1
+
         self.internal_application_port = (
             internal_application_port
             or mlrun.mlconf.function.application.default_sidecar_internal_port
@@ -168,12 +174,13 @@ class ApplicationStatus(NuclioStatus):
         self.application_source = application_source or None
         self.sidecar_name = sidecar_name or None
         self.api_gateway_name = api_gateway_name or None
-        self.api_gateway = api_gateway or None
+        self.api_gateway: typing.Optional[APIGateway] = api_gateway or None
         self.url = url or None
 
 
 class ApplicationRuntime(RemoteRuntime):
     kind = "application"
+    reverse_proxy_image = None
 
     @min_nuclio_versions("1.13.1")
     def __init__(self, spec=None, metadata=None):
@@ -252,6 +259,15 @@ class ApplicationRuntime(RemoteRuntime):
                     "Application sidecar spec must include a command if args are provided"
                 )
 
+    def prepare_image_for_deploy(self):
+        if self.spec.build.source and self.spec.build.load_source_on_run:
+            logger.warning(
+                "Application runtime requires loading the source into the application image. "
+                f"Even though {self.spec.build.load_source_on_run=}, loading on build will be forced."
+            )
+            self.spec.build.load_source_on_run = False
+        super().prepare_image_for_deploy()
+
     def deploy(
         self,
         project="",
@@ -273,6 +289,7 @@ class ApplicationRuntime(RemoteRuntime):
         """
         Deploy function, builds the application image if required (self.requires_build()) or force_build is True,
         Once the image is built, the function is deployed.
+
         :param project:                 Project name
         :param tag:                     Function tag
         :param verbose:                 Set True for verbose logging
@@ -306,10 +323,11 @@ class ApplicationRuntime(RemoteRuntime):
                 show_on_failure=show_on_failure,
             )
 
-        self._ensure_reverse_proxy_configurations()
+        # This is a class method that accepts a function instance, so we pass self as the function instance
+        self._ensure_reverse_proxy_configurations(self)
         self._configure_application_sidecar()
 
-        # we only allow accessing the application via the API Gateway
+        # We only allow accessing the application via the API Gateway
         name_tag = tag or self.metadata.tag
         self.status.api_gateway_name = (
             f"{self.metadata.name}-{name_tag}" if name_tag else self.metadata.name
@@ -346,9 +364,13 @@ class ApplicationRuntime(RemoteRuntime):
         )
 
     def with_source_archive(
-        self, source, workdir=None, pull_at_runtime=True, target_dir=None
+        self,
+        source,
+        workdir=None,
+        pull_at_runtime: bool = False,
+        target_dir: str = None,
     ):
-        """load the code from git/tar/zip archive at runtime or build
+        """load the code from git/tar/zip archive at build
 
         :param source:          valid absolute path or URL to git, zip, or tar file, e.g.
                                 git://github.com/mlrun/something.git
@@ -356,13 +378,20 @@ class ApplicationRuntime(RemoteRuntime):
                                 note path source must exist on the image or exist locally when run is local
                                 (it is recommended to use 'workdir' when source is a filepath instead)
         :param workdir:         working dir relative to the archive root (e.g. './subdir') or absolute to the image root
-        :param pull_at_runtime: load the archive into the container at job runtime vs on build/deploy
+        :param pull_at_runtime: currently not supported, source must be loaded into the image during the build process
         :param target_dir:      target dir on runtime pod or repo clone / archive extraction
         """
+        if pull_at_runtime:
+            logger.warning(
+                f"{pull_at_runtime=} is currently not supported for application runtime "
+                "and will be overridden to False",
+                pull_at_runtime=pull_at_runtime,
+            )
+
         self._configure_mlrun_build_with_source(
             source=source,
             workdir=workdir,
-            pull_at_runtime=pull_at_runtime,
+            pull_at_runtime=False,
             target_dir=target_dir,
         )
 
@@ -391,8 +420,8 @@ class ApplicationRuntime(RemoteRuntime):
             "main:Handler",
         )
 
-    @classmethod
-    def get_filename_and_handler(cls) -> (str, str):
+    @staticmethod
+    def get_filename_and_handler() -> (str, str):
         reverse_proxy_file_path = pathlib.Path(__file__).parent / "reverse_proxy.go"
         return str(reverse_proxy_file_path), "Handler"
 
@@ -488,6 +517,39 @@ class ApplicationRuntime(RemoteRuntime):
             **http_client_kwargs,
         )
 
+    @classmethod
+    def deploy_reverse_proxy_image(cls):
+        """
+        Build the reverse proxy image and save it.
+        The reverse proxy image is used to route requests to the application sidecar.
+        This is useful when you want to decrease build time by building the application image only once.
+
+        :param use_cache:   Use the cache when building the image
+        """
+        # create a function that includes only the reverse proxy, without the application
+
+        reverse_proxy_func = mlrun.run.new_function(
+            name="reverse-proxy-temp", kind="remote"
+        )
+        # default max replicas is 4, we only need one replica for the reverse proxy
+        reverse_proxy_func.spec.max_replicas = 1
+
+        # the reverse proxy image should not be based on another image
+        reverse_proxy_func.set_config("spec.build.baseImage", None)
+        reverse_proxy_func.spec.image = ""
+        reverse_proxy_func.spec.build.base_image = ""
+
+        cls._ensure_reverse_proxy_configurations(reverse_proxy_func)
+        reverse_proxy_func.deploy()
+
+        # save the created container image
+        cls.reverse_proxy_image = reverse_proxy_func.status.container_image
+
+        # delete the function to avoid cluttering the project
+        mlrun.get_run_db().delete_function(
+            reverse_proxy_func.metadata.name, reverse_proxy_func.metadata.project
+        )
+
     def _run(self, runobj: "mlrun.RunObject", execution):
         raise mlrun.runtimes.RunError(
             "Application runtime .run() is not yet supported. Use .invoke() instead."
@@ -515,6 +577,13 @@ class ApplicationRuntime(RemoteRuntime):
                 args=self.spec.args,
             )
 
+        if self.spec.build.source in [".", "./"]:
+            logger.info(
+                "The application is configured to use the project's source. "
+                "Application runtime requires loading the source into the application image. "
+                "Loading on build will be forced regardless of whether 'pull_at_runtime=True' was configured."
+            )
+
         with_mlrun = self._resolve_build_with_mlrun(with_mlrun)
         return self._build_image(
             builder_env=builder_env,
@@ -527,21 +596,29 @@ class ApplicationRuntime(RemoteRuntime):
             with_mlrun=with_mlrun,
         )
 
-    def _ensure_reverse_proxy_configurations(self):
-        if self.spec.build.functionSourceCode or self.status.container_image:
+    @staticmethod
+    def _ensure_reverse_proxy_configurations(function: RemoteRuntime):
+        if function.spec.build.functionSourceCode or function.status.container_image:
             return
 
         filename, handler = ApplicationRuntime.get_filename_and_handler()
         name, spec, code = nuclio.build_file(
             filename,
-            name=self.metadata.name,
+            name=function.metadata.name,
             handler=handler,
         )
-        self.spec.function_handler = mlrun.utils.get_in(spec, "spec.handler")
-        self.spec.build.functionSourceCode = mlrun.utils.get_in(
+        function.spec.function_handler = mlrun.utils.get_in(spec, "spec.handler")
+        function.spec.build.functionSourceCode = mlrun.utils.get_in(
             spec, "spec.build.functionSourceCode"
         )
-        self.spec.nuclio_runtime = mlrun.utils.get_in(spec, "spec.runtime")
+        function.spec.nuclio_runtime = mlrun.utils.get_in(spec, "spec.runtime")
+
+        # default the reverse proxy logger level to info
+        logger_sinks_key = "spec.loggerSinks"
+        if not function.spec.config.get(logger_sinks_key):
+            function.set_config(
+                logger_sinks_key, [{"level": "info", "sink": "myStdoutLoggerSink"}]
+            )
 
     def _configure_application_sidecar(self):
         # Save the application image in the status to allow overriding it with the reverse proxy entry point
@@ -552,8 +629,12 @@ class ApplicationRuntime(RemoteRuntime):
             self.status.application_image = self.spec.image
             self.spec.image = ""
 
-        if self.status.container_image:
-            self.from_image(self.status.container_image)
+        # reuse the reverse proxy image if it was built before
+        if (
+            reverse_proxy_image := self.status.container_image
+            or self.reverse_proxy_image
+        ):
+            self.from_image(reverse_proxy_image)
 
         self.status.sidecar_name = f"{self.metadata.name}-sidecar"
         self.with_sidecar(

mlrun/runtimes/nuclio/function.py

@@ -689,7 +689,7 @@ class RemoteRuntime(KubeResource):
             "State thresholds do not apply for nuclio as it has its own function pods healthiness monitoring"
         )
 
-    @min_nuclio_versions("1.12.8")
+    @min_nuclio_versions("1.13.1")
     def disable_default_http_trigger(
         self,
     ):
@@ -707,6 +707,10 @@ class RemoteRuntime(KubeResource):
         """
         self.spec.disable_default_http_trigger = False
 
+    def skip_image_enrichment(self):
+        # make sure the API does not enrich the base image if the function is not a python function
+        return self.spec.nuclio_runtime and "python" not in self.spec.nuclio_runtime
+
     def _get_state(
         self,
         dashboard="",

mlrun/runtimes/pod.py

@@ -1174,9 +1174,9 @@ class KubeResource(BaseRuntime, KfpAdapterMixin):
         """
         if node_name:
             self.spec.node_name = node_name
-        if node_selector:
+        if node_selector is not None:
             self.spec.node_selector = node_selector
-        if affinity:
+        if affinity is not None:
             self.spec.affinity = affinity
         if tolerations is not None:
             self.spec.tolerations = tolerations

mlrun/runtimes/remotesparkjob.py

@@ -102,16 +102,13 @@ class RemoteSparkRuntime(KubejobRuntime):
 
     @classmethod
     def deploy_default_image(cls):
-        from mlrun import get_run_db
-        from mlrun.run import new_function
-
-        sj = new_function(
+        sj = mlrun.new_function(
             kind="remote-spark", name="remote-spark-default-image-deploy-temp"
         )
         sj.spec.build.image = cls.default_image
         sj.with_spark_service(spark_service="dummy-spark")
         sj.deploy()
-        get_run_db().delete_function(name=sj.metadata.name)
+        mlrun.get_run_db().delete_function(name=sj.metadata.name)
 
     def is_deployed(self):
         if (

mlrun/runtimes/sparkjob/spark3job.py

@@ -505,13 +505,10 @@ class Spark3Runtime(KubejobRuntime):
             raise NotImplementedError(
                 "Setting node name is not supported for spark runtime"
             )
-        # TODO add affinity support
-        # https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go#L491
-        if affinity:
-            raise NotImplementedError(
-                "Setting affinity is not supported for spark runtime"
-            )
-        super().with_node_selection(node_name, node_selector, affinity, tolerations)
+        self.with_driver_node_selection(node_name, node_selector, affinity, tolerations)
+        self.with_executor_node_selection(
+            node_name, node_selector, affinity, tolerations
+        )
 
     def with_driver_node_selection(
         self,
@@ -537,11 +534,11 @@ class Spark3Runtime(KubejobRuntime):
             raise NotImplementedError(
                 "Setting node name is not supported for spark runtime"
             )
-        if affinity:
+        if affinity is not None:
             self.spec.driver_affinity = affinity
-        if node_selector:
+        if node_selector is not None:
             self.spec.driver_node_selector = node_selector
-        if tolerations:
+        if tolerations is not None:
             self.spec.driver_tolerations = tolerations
 
     def with_executor_node_selection(
@@ -568,11 +565,11 @@ class Spark3Runtime(KubejobRuntime):
             raise NotImplementedError(
                 "Setting node name is not supported for spark runtime"
             )
-        if affinity:
+        if affinity is not None:
             self.spec.executor_affinity = affinity
-        if node_selector:
+        if node_selector is not None:
             self.spec.executor_node_selector = node_selector
-        if tolerations:
+        if tolerations is not None:
             self.spec.executor_tolerations = tolerations
 
     def with_preemption_mode(
@@ -811,9 +808,7 @@ class Spark3Runtime(KubejobRuntime):
 
     @classmethod
     def deploy_default_image(cls, with_gpu=False):
-        from mlrun.run import new_function
-
-        sj = new_function(kind=cls.kind, name="spark-default-image-deploy-temp")
+        sj = mlrun.new_function(kind=cls.kind, name="spark-default-image-deploy-temp")
         sj.spec.build.image = cls._get_default_deployed_mlrun_image_name(with_gpu)
 
         # setting required resources

mlrun/serving/routers.py

@@ -32,7 +32,6 @@ from mlrun.errors import err_to_str
 from mlrun.utils import logger, now_date
 
 from ..common.helpers import parse_versioned_object_uri
-from ..config import config
 from .server import GraphServer
 from .utils import RouterToDict, _extract_input_data, _update_result_body
 from .v2_serving import _ModelLogPusher
@@ -1057,9 +1056,7 @@ def _init_endpoint_record(
                 function_uri=graph_server.function_uri,
                 model=versioned_model_name,
                 model_class=voting_ensemble.__class__.__name__,
-                stream_path=config.model_endpoint_monitoring.store_prefixes.default.format(
-                    project=project, kind="stream"
-                ),
+                stream_path=voting_ensemble.context.stream.stream_uri,
                 active=True,
                 monitoring_mode=mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled,
             ),

mlrun/serving/server.py

@@ -38,10 +38,7 @@ from ..errors import MLRunInvalidArgumentError
 from ..model import ModelObj
 from ..utils import get_caller_globals
 from .states import RootFlowStep, RouterStep, get_function, graph_root_setter
-from .utils import (
-    event_id_key,
-    event_path_key,
-)
+from .utils import event_id_key, event_path_key
 
 
 class _StreamContext:
@@ -71,15 +68,15 @@ class _StreamContext:
                 function_uri, config.default_project
             )
 
-            stream_uri = mlrun.model_monitoring.get_stream_path(project=project)
+            self.stream_uri = mlrun.model_monitoring.get_stream_path(project=project)
 
             if log_stream:
                 # Update the stream path to the log stream value
-                stream_uri = log_stream.format(project=project)
+                self.stream_uri = log_stream.format(project=project)
 
             stream_args = parameters.get("stream_args", {})
 
-            self.output_stream = get_stream_pusher(stream_uri, **stream_args)
+            self.output_stream = get_stream_pusher(self.stream_uri, **stream_args)
 
 
 class GraphServer(ModelObj):

mlrun/serving/states.py

@@ -1391,7 +1391,7 @@ class FlowStep(BaseStep):
         return step
 
     def supports_termination(self):
-        return self.engine == "async"
+        return self.engine != "sync"
 
 
 class RootFlowStep(FlowStep):