mlrun 1.10.0rc15__tar.gz → 1.10.0rc16__tar.gz

{mlrun-1.10.0rc15/mlrun.egg-info → mlrun-1.10.0rc16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.10.0rc15
+Version: 1.10.0rc16
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv

{mlrun-1.10.0rc15 → mlrun-1.10.0rc16}/mlrun/artifacts/llm_prompt.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 import json
 import tempfile
+from collections import defaultdict
 from typing import Optional, Union
 
 import mlrun
@@ -253,3 +254,8 @@ class LLMPromptArtifact(Artifact):
             self.spec.prompt_template = None
             self._src_is_temp = True
         super().before_log()
+
+
+class PlaceholderDefaultDict(defaultdict):
+    def __missing__(self, key):
+        return f"{{{key}}}"

{mlrun-1.10.0rc15 → mlrun-1.10.0rc16}/mlrun/common/constants.py

@@ -83,6 +83,7 @@ class MLRunInternalLabels:
     mlrun_type = "mlrun__type"
     original_workflow_id = "original-workflow-id"
     workflow_id = "workflow-id"
+    retrying = "retrying"
 
     owner = "owner"
     v3io_user = "v3io_user"

{mlrun-1.10.0rc15 → mlrun-1.10.0rc16}/mlrun/common/schemas/workflow.py

@@ -51,6 +51,8 @@ class RerunWorkflowRequest(pydantic.v1.BaseModel):
     run_id: typing.Optional[str] = None
     notifications: typing.Optional[list[Notification]] = None
     workflow_runner_node_selector: typing.Optional[dict[str, str]] = None
+    original_workflow_runner_uid: typing.Optional[str] = None
+    original_workflow_name: typing.Optional[str] = None
 
 
 class WorkflowResponse(pydantic.v1.BaseModel):

{mlrun-1.10.0rc15 → mlrun-1.10.0rc16}/mlrun/db/base.py

@@ -55,6 +55,12 @@ class RunDBInterface(ABC):
     def update_run(self, updates: dict, uid, project="", iter=0):
         pass
 
+    @abstractmethod
+    def set_run_retrying_status(
+        self, project: str, name: str, run_id: str, retrying: bool
+    ):
+        pass
+
     @abstractmethod
     def abort_run(self, uid, project="", iter=0, timeout=45, status_text=""):
         pass

{mlrun-1.10.0rc15 → mlrun-1.10.0rc16}/mlrun/db/httpdb.py

@@ -4741,6 +4741,28 @@ class HTTPRunDB(RunDBInterface):
         )
         return mlrun.common.schemas.GetWorkflowResponse(**response.json())
 
+    def set_run_retrying_status(
+        self, project: str, name: str, run_id: str, retrying: bool = False
+    ):
+        """
+        Toggle the “retrying” label on a workflow-runner run.
+
+        This will POST to the workflows endpoint to either add or remove the
+        `retrying` flag on a specific run, which prevents parallel retries.
+
+        :param project:   The project name under which the workflow is defined.
+        :param name:      The workflow name (as in the URL path).
+        :param run_id:    The UID of the workflow-runner run to update.
+        :param retrying:  True to add the `retrying` label, False to remove it.
+
+        :raises MLRunHTTPError: If the HTTP request fails or returns an error status.
+        """
+        path = f"projects/{project}/workflows/{name}/runs/{run_id}/set-retry-status"
+        params = {"retrying": retrying}
+        self.api_call(
+            "POST", path, f"set retrying on {project}/{run_id}", params=params
+        )
+
     def load_project(
         self,
         name: str,

{mlrun-1.10.0rc15 → mlrun-1.10.0rc16}/mlrun/db/nopdb.py

@@ -72,6 +72,11 @@ class NopDB(RunDBInterface):
     def update_run(self, updates: dict, uid, project="", iter=0):
         pass
 
+    def set_run_retrying_status(
+        self, project: str, name: str, run_id: str, retrying: bool
+    ):
+        pass
+
     def abort_run(self, uid, project="", iter=0, timeout=45, status_text=""):
         pass
 

{mlrun-1.10.0rc15 → mlrun-1.10.0rc16}/mlrun/launcher/local.py

@@ -276,6 +276,19 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
                     args = sp[1:]
         return command, args
 
+    def _validate_run(
+        self,
+        runtime: "mlrun.runtimes.BaseRuntime",
+        run: "mlrun.run.RunObject",
+    ):
+        super()._validate_run(runtime, run)
+        if self._is_run_local and run.spec.retry.count:
+            logger.warning(
+                "Retry is not supported for local runs, ignoring retry settings",
+                retry=run.spec.retry.to_dict(),
+            )
+            run.spec.retry.count = 0
+
     def _push_notifications(
         self, runobj: "mlrun.run.RunObject", runtime: "mlrun.runtimes.BaseRuntime"
     ):

{mlrun-1.10.0rc15 → mlrun-1.10.0rc16}/mlrun/projects/pipelines.py

@@ -1072,7 +1072,11 @@ def github_webhook(request):
 
 
 def rerun_workflow(
-    context: mlrun.execution.MLClientCtx, run_uid: str, project_name: str
+    context: mlrun.execution.MLClientCtx,
+    run_uid: str,
+    project_name: str,
+    original_runner_uid: str,
+    original_workflow_name: str,
 ):
     """
     Re-run a workflow by retrying a previously failed KFP pipeline.
@@ -1080,8 +1084,11 @@ def rerun_workflow(
     :param context:      MLRun context.
     :param run_uid:      The run UID of the original workflow to retry.
     :param project_name: The project name.
+    :param original_runner_uid: The original workflow runner UID.
+    :param original_workflow_name: The original workflow name.
     """
     db = mlrun.get_run_db()
+    new_pipeline_id = None
 
     try:
         # Invoke the KFP retry endpoint (direct-submit mode)
@@ -1096,6 +1103,24 @@ def rerun_workflow(
             rerun_of_workflow=run_uid,
         )
 
+        # Enqueue "running" notifications server-side for this RerunRunner run
+        db.push_run_notifications(context.uid, project_name)
+
+        context.set_label(
+            mlrun_constants.MLRunInternalLabels.workflow_id, new_pipeline_id
+        )
+        context.update_run()
+
+        context.log_result("workflow_id", new_pipeline_id)
+
+        pipeline = wait_for_pipeline_completion(
+            new_pipeline_id,
+            project=project_name,
+        )
+
+        final_state = pipeline["run"]["status"]
+        context.log_result("workflow_state", final_state, commit=True)
+
     except mlrun.errors.MLRunHTTPError as http_exc:
         logger.error(
             "Failed calling KFP retry API",
@@ -1104,33 +1129,28 @@ def rerun_workflow(
         )
         raise
 
-    # Enqueue "running" notifications server-side for this RerunRunner run
-    db.push_run_notifications(context.uid, project_name)
-
-    context.set_label(mlrun_constants.MLRunInternalLabels.workflow_id, new_pipeline_id)
-    context.update_run()
-
-    context.log_result("workflow_id", new_pipeline_id)
-
-    try:
-        pipeline = wait_for_pipeline_completion(
-            new_pipeline_id,
-            project=project_name,
-        )
     except Exception as exc:
-        mlrun.utils.logger.error(
-            "Failed waiting for workflow completion",
+        logger.error(
+            "Error during rerun_workflow execution",
+            error=err_to_str(exc),
             rerun_pipeline_id=new_pipeline_id,
-            exc=err_to_str(exc),
         )
-    else:
-        final_state = pipeline["run"]["status"]
-        context.log_result("workflow_state", final_state, commit=True)
+        raise
 
-        if final_state != mlrun_pipelines.common.models.RunStatuses.succeeded:
-            raise mlrun.errors.MLRunRuntimeError(
-                f"Pipeline retry of {run_uid} finished in state={final_state}"
-            )
+    finally:
+        # Once the rerun has finished, clear the “retrying” label on the original runner
+        # so that subsequent retry requests can acquire the lock again.
+        db.set_run_retrying_status(
+            project=project_name,
+            name=original_workflow_name,
+            run_id=original_runner_uid,
+            retrying=False,
+        )
+
+    if final_state != mlrun_pipelines.common.models.RunStatuses.succeeded:
+        raise mlrun.errors.MLRunRuntimeError(
+            f"Pipeline retry of {run_uid} finished in state={final_state}"
+        )
 
 
 def load_and_run(context, *args, **kwargs):

{mlrun-1.10.0rc15 → mlrun-1.10.0rc16}/mlrun/projects/project.py

@@ -5073,7 +5073,6 @@ class MlrunProject(ModelObj):
         :param states: List only runs whose state is one of the provided states.
         :param sort: Whether to sort the result according to their start time. Otherwise, results will be
             returned by their internal order in the DB (order will not be guaranteed).
-        :param last: Deprecated - currently not used (will be removed in 1.10.0).
         :param iter: If ``True`` return runs from all iterations. Otherwise, return only runs whose ``iter`` is 0.
         :param start_time_from: Filter by run start time in ``[start_time_from, start_time_to]``.
         :param start_time_to: Filter by run start time in ``[start_time_from, start_time_to]``.

{mlrun-1.10.0rc15 → mlrun-1.10.0rc16}/mlrun/runtimes/utils.py

@@ -445,8 +445,6 @@ def enrich_run_labels(
     labels_enrichment = {
         mlrun_constants.MLRunInternalLabels.owner: os.environ.get("V3IO_USERNAME")
         or getpass.getuser(),
-        # TODO: remove this in 1.10.0
-        mlrun_constants.MLRunInternalLabels.v3io_user: os.environ.get("V3IO_USERNAME"),
     }
 
     # Resolve which label keys to enrich

{mlrun-1.10.0rc15 → mlrun-1.10.0rc16}/mlrun/serving/server.py

@@ -350,33 +350,33 @@ def add_error_raiser_step(
     monitored_steps_raisers = {}
     user_steps = list(graph.steps.values())
     for monitored_step in monitored_steps.values():
-        if monitored_step.raise_exception:
-            error_step = graph.add_step(
-                class_name="mlrun.serving.states.ModelRunnerErrorRaiser",
-                name=f"{monitored_step.name}_error_raise",
-                after=monitored_step.name,
-                full_event=True,
-                raise_exception=monitored_step.raise_exception,
-                models_names=list(monitored_step.class_args["models"].keys()),
-                model_endpoint_creation_strategy=mlrun.common.schemas.ModelEndpointCreationStrategy.SKIP,
-            )
-            if monitored_step.responder:
-                monitored_step.responder = False
-                error_step.respond()
-            monitored_steps_raisers[monitored_step.name] = error_step.name
-            error_step.on_error = monitored_step.on_error
-    for step in user_steps:
-        if step.after:
-            if isinstance(step.after, list):
-                for i in range(len(step.after)):
-                    if step.after[i] in monitored_steps_raisers:
-                        step.after[i] = monitored_steps_raisers[step.after[i]]
-            else:
-                if (
-                    isinstance(step.after, str)
-                    and step.after in monitored_steps_raisers
-                ):
-                    step.after = monitored_steps_raisers[step.after]
+        error_step = graph.add_step(
+            class_name="mlrun.serving.states.ModelRunnerErrorRaiser",
+            name=f"{monitored_step.name}_error_raise",
+            after=monitored_step.name,
+            full_event=True,
+            raise_exception=monitored_step.raise_exception,
+            models_names=list(monitored_step.class_args["models"].keys()),
+            model_endpoint_creation_strategy=mlrun.common.schemas.ModelEndpointCreationStrategy.SKIP,
+        )
+        if monitored_step.responder:
+            monitored_step.responder = False
+            error_step.respond()
+        monitored_steps_raisers[monitored_step.name] = error_step.name
+        error_step.on_error = monitored_step.on_error
+    if monitored_steps_raisers:
+        for step in user_steps:
+            if step.after:
+                if isinstance(step.after, list):
+                    for i in range(len(step.after)):
+                        if step.after[i] in monitored_steps_raisers:
+                            step.after[i] = monitored_steps_raisers[step.after[i]]
+                else:
+                    if (
+                        isinstance(step.after, str)
+                        and step.after in monitored_steps_raisers
+                    ):
+                        step.after = monitored_steps_raisers[step.after]
     return graph
 
 

{mlrun-1.10.0rc15 → mlrun-1.10.0rc16}/mlrun/serving/states.py

@@ -35,7 +35,7 @@ from storey import ParallelExecutionMechanisms
 import mlrun
 import mlrun.artifacts
 import mlrun.common.schemas as schemas
-from mlrun.artifacts.llm_prompt import LLMPromptArtifact
+from mlrun.artifacts.llm_prompt import LLMPromptArtifact, PlaceholderDefaultDict
 from mlrun.artifacts.model import ModelArtifact
 from mlrun.datastore.datastore_profile import (
     DatastoreProfileKafkaSource,
@@ -45,7 +45,7 @@ from mlrun.datastore.datastore_profile import (
 )
 from mlrun.datastore.model_provider.model_provider import ModelProvider
 from mlrun.datastore.storeytargets import KafkaStoreyTarget, StreamStoreyTarget
-from mlrun.utils import logger
+from mlrun.utils import get_data_from_path, logger, split_path
 
 from ..config import config
 from ..datastore import get_stream_pusher
@@ -501,10 +501,15 @@ class BaseStep(ModelObj):
     def verify_model_runner_step(
         self,
         step: "ModelRunnerStep",
+        step_model_endpoints_names: Optional[list[str]] = None,
+        verify_shared_models: bool = True,
     ):
         """
         Verify ModelRunnerStep, can be part of Flow graph and models can not repeat in graph.
-        :param step: ModelRunnerStep to verify
+        :param step:                        ModelRunnerStep to verify
+        :param step_model_endpoints_names:  List of model endpoints names that are in the step.
+                                            if provided will ignore step models and verify only the models on list.
+        :param verify_shared_models:        If True, verify that shared models are defined in the graph.
         """
 
         if not isinstance(step, ModelRunnerStep):
@@ -516,7 +521,7 @@ class BaseStep(ModelObj):
             raise GraphError(
                 "ModelRunnerStep can be added to 'Flow' topology graph only"
             )
-        step_model_endpoints_names = list(
+        step_model_endpoints_names = step_model_endpoints_names or list(
             step.class_args.get(schemas.ModelRunnerStepData.MODELS, {}).keys()
         )
         # Get all model_endpoints names that are in both lists
@@ -530,8 +535,9 @@ class BaseStep(ModelObj):
                 f"The graph already contains the model endpoints named - {common_endpoints_names}."
             )
 
-        # Check if shared models are defined in the graph
-        self._verify_shared_models(root, step, step_model_endpoints_names)
+        if verify_shared_models:
+            # Check if shared models are defined in the graph
+            self._verify_shared_models(root, step, step_model_endpoints_names)
         # Update model endpoints names in the root step
         root.update_model_endpoints_names(step_model_endpoints_names)
 
@@ -569,7 +575,9 @@ class BaseStep(ModelObj):
                 llm_artifact, _ = mlrun.store_manager.get_store_artifact(
                     model_artifact_uri
                 )
-                model_artifact_uri = llm_artifact.spec.parent_uri
+                model_artifact_uri = mlrun.utils.remove_tag_from_artifact_uri(
+                    llm_artifact.spec.parent_uri
+                )
             actual_shared_name = root.get_shared_model_name_by_artifact_uri(
                 model_artifact_uri
             )
@@ -1148,11 +1156,11 @@ class Model(storey.ParallelExecutionRunnable, ModelObj):
     def init(self):
         self.load()
 
-    def predict(self, body: Any) -> Any:
+    def predict(self, body: Any, **kwargs) -> Any:
         """Override to implement prediction logic. If the logic requires asyncio, override predict_async() instead."""
         return body
 
-    async def predict_async(self, body: Any) -> Any:
+    async def predict_async(self, body: Any, **kwargs) -> Any:
         """Override to implement prediction logic if the logic requires asyncio."""
         return body
 
@@ -1197,11 +1205,18 @@ class Model(storey.ParallelExecutionRunnable, ModelObj):
 
 
 class LLModel(Model):
-    def __init__(self, name: str, **kwargs):
+    def __init__(
+        self, name: str, input_path: Optional[Union[str, list[str]]], **kwargs
+    ):
         super().__init__(name, **kwargs)
+        self._input_path = split_path(input_path)
 
     def predict(
-        self, body: Any, messages: list[dict], model_configuration: dict
+        self,
+        body: Any,
+        messages: Optional[list[dict]] = None,
+        model_configuration: Optional[dict] = None,
+        **kwargs,
     ) -> Any:
         if isinstance(
             self.invocation_artifact, mlrun.artifacts.LLMPromptArtifact
@@ -1214,7 +1229,11 @@ class LLModel(Model):
         return body
 
     async def predict_async(
-        self, body: Any, messages: list[dict], model_configuration: dict
+        self,
+        body: Any,
+        messages: Optional[list[dict]] = None,
+        model_configuration: Optional[dict] = None,
+        **kwargs,
     ) -> Any:
         if isinstance(
             self.invocation_artifact, mlrun.artifacts.LLMPromptArtifact
@@ -1262,12 +1281,34 @@ class LLModel(Model):
             return None, None
         prompt_legend = llm_prompt_artifact.spec.prompt_legend
         prompt_template = deepcopy(llm_prompt_artifact.read_prompt())
-        kwargs = {
-            place_holder: body.get(body_map["field"])
-            for place_holder, body_map in prompt_legend.items()
-        }
-        for d in prompt_template:
-            d["content"] = d["content"].format(**kwargs)
+        input_data = copy(get_data_from_path(self._input_path, body))
+        if isinstance(input_data, dict):
+            kwargs = (
+                {
+                    place_holder: input_data.get(body_map["field"])
+                    for place_holder, body_map in prompt_legend.items()
+                }
+                if prompt_legend
+                else {}
+            )
+            input_data.update(kwargs)
+            default_place_holders = PlaceholderDefaultDict(lambda: None, input_data)
+            for message in prompt_template:
+                try:
+                    message["content"] = message["content"].format(**input_data)
+                except KeyError as e:
+                    logger.warning(
+                        "Input data was missing a placeholder, placeholder stay unformatted",
+                        key_error=e,
+                    )
+                    message["content"] = message["content"].format_map(
+                        default_place_holders
+                    )
+        else:
+            logger.warning(
+                f"Expected input data to be a dict, but received input data from type {type(input_data)} prompt "
+                f"template stay unformatted",
+            )
         return prompt_template, llm_prompt_artifact.spec.model_configuration
 
 
@@ -1567,11 +1608,27 @@ class ModelRunnerStep(MonitoredStep):
           :param outputs:             list of the model outputs (e.g. labels) ,if provided will override the outputs
                                       that been configured in the model artifact, please note that those outputs need to
                                       be equal to the model_class predict method outputs (length, and order)
-          :param input_path:          input path inside the user event, expect scopes to be defined by dot notation
-                                      (e.g "inputs.my_model_inputs"). expects list or dictionary type object in path.
-          :param result_path:         result path inside the user output event, expect scopes to be defined by dot
-                                      notation (e.g "outputs.my_model_outputs") expects list or dictionary type object
-                                      in path.
+          :param input_path:          when specified selects the key/path in the event to use as model monitoring inputs
+                                      this require that the event body will behave like a dict, expects scopes to be
+                                      defined by dot notation (e.g "data.d").
+                                      examples: input_path="data.b"
+                                      event: {"data":{"a": 5, "b": 7}}, means monitored body will be 7.
+                                      event: {"data":{"a": [5, 9], "b": [7, 8]}} means monitored body will be [7,8].
+                                      event: {"data":{"a": "extra_data", "b": {"f0": [1, 2]}}} means monitored body will
+                                      be {"f0": [1, 2]}.
+                                      if a ``list`` or ``list of lists`` is provided, it must follow the order and
+                                      size defined by the input schema.
+          :param result_path:         when specified selects the key/path in the output event to use as model monitoring
+                                      outputs this require that the output event body will behave like a dict,
+                                      expects scopes to be defined by dot notation (e.g "data.d").
+                                      examples: result_path="out.b"
+                                      event: {"out":{"a": 5, "b": 7}}, means monitored body will be 7.
+                                      event: {"out":{"a": [5, 9], "b": [7, 8]}} means monitored body will be [7,8]
+                                      event: {"out":{"a": "extra_data", "b": {"f0": [1, 2]}}} means monitored body will
+                                      be {"f0": [1, 2]}
+                                      if a ``list`` or ``list of lists`` is provided, it must follow the order and
+                                      size defined by the output schema.
+
           :param override:            bool allow override existing model on the current ModelRunnerStep.
           :param model_parameters:    Parameters for model instantiation
         """
@@ -1590,7 +1647,7 @@ class ModelRunnerStep(MonitoredStep):
         ):
             try:
                 model_artifact, _ = mlrun.store_manager.get_store_artifact(
-                    model_artifact
+                    mlrun.utils.remove_tag_from_artifact_uri(model_artifact)
                 )
             except mlrun.errors.MLRunNotFoundError:
                 raise mlrun.errors.MLRunInvalidArgumentError("Artifact not found.")
@@ -1602,6 +1659,11 @@ class ModelRunnerStep(MonitoredStep):
             if isinstance(model_artifact, mlrun.artifacts.Artifact)
             else model_artifact
         )
+        model_artifact = (
+            mlrun.utils.remove_tag_from_artifact_uri(model_artifact)
+            if model_artifact
+            else None
+        )
         model_parameters["artifact_uri"] = model_parameters.get(
             "artifact_uri", model_artifact
         )
@@ -1617,6 +1679,11 @@ class ModelRunnerStep(MonitoredStep):
             raise mlrun.errors.MLRunInvalidArgumentError(
                 f"Model with name {endpoint_name} already exists in this ModelRunnerStep."
             )
+        root = self._extract_root_step()
+        if isinstance(root, RootFlowStep):
+            self.verify_model_runner_step(
+                self, [endpoint_name], verify_shared_models=False
+            )
         ParallelExecutionMechanisms.validate(execution_mechanism)
         self.class_args[schemas.ModelRunnerStepData.MODEL_TO_EXECUTION_MECHANISM] = (
             self.class_args.get(
@@ -1693,15 +1760,6 @@ class ModelRunnerStep(MonitoredStep):
             )
         return output_schema
 
-    @staticmethod
-    def _split_path(path: str) -> Union[str, list[str], None]:
-        if path is not None:
-            parsed_path = path.split(".")
-            if len(parsed_path) == 1:
-                parsed_path = parsed_path[0]
-            return parsed_path
-        return path
-
     def _calculate_monitoring_data(self) -> dict[str, dict[str, str]]:
         monitoring_data = deepcopy(
             self.class_args.get(
@@ -1726,15 +1784,11 @@ class ModelRunnerStep(MonitoredStep):
                 ][model][schemas.MonitoringData.OUTPUTS] = monitoring_data[model][
                     schemas.MonitoringData.OUTPUTS
                 ]
-                monitoring_data[model][schemas.MonitoringData.INPUT_PATH] = (
-                    self._split_path(
-                        monitoring_data[model][schemas.MonitoringData.INPUT_PATH]
-                    )
+                monitoring_data[model][schemas.MonitoringData.INPUT_PATH] = split_path(
+                    monitoring_data[model][schemas.MonitoringData.INPUT_PATH]
                 )
-                monitoring_data[model][schemas.MonitoringData.RESULT_PATH] = (
-                    self._split_path(
-                        monitoring_data[model][schemas.MonitoringData.RESULT_PATH]
-                    )
+                monitoring_data[model][schemas.MonitoringData.RESULT_PATH] = split_path(
+                    monitoring_data[model][schemas.MonitoringData.RESULT_PATH]
                 )
             return monitoring_data
 
@@ -1752,6 +1806,13 @@ class ModelRunnerStep(MonitoredStep):
             model_selector = get_class(model_selector, namespace)()
         model_objects = []
         for model, model_params in models.values():
+            model_params[schemas.MonitoringData.INPUT_PATH] = (
+                self.class_args.get(
+                    mlrun.common.schemas.ModelRunnerStepData.MONITORING_DATA, {}
+                )
+                .get(model_params.get("name"), {})
+                .get(schemas.MonitoringData.INPUT_PATH)
+            )
             model = get_class(model, namespace).from_dict(
                 model_params, init_with_params=True
             )
@@ -2401,7 +2462,13 @@ class FlowStep(BaseStep):
         if not step.before and not any(
             [step.name in other_step.after for other_step in self._steps.values()]
         ):
-            step.responder = True
+            if any(
+                [
+                    getattr(step_in_graph, "responder", False)
+                    for step_in_graph in self._steps.values()
+                ]
+            ):
+                step.responder = True
             return
 
         for step_name in step.before:
@@ -2484,7 +2551,7 @@ class RootFlowStep(FlowStep):
         name: str,
         model_class: Union[str, Model],
         execution_mechanism: Union[str, ParallelExecutionMechanisms],
-        model_artifact: Optional[Union[str, ModelArtifact]],
+        model_artifact: Union[str, ModelArtifact],
         override: bool = False,
         **model_parameters,
     ) -> None:
@@ -2536,6 +2603,7 @@ class RootFlowStep(FlowStep):
             if isinstance(model_artifact, mlrun.artifacts.Artifact)
             else model_artifact
         )
+        model_artifact = mlrun.utils.remove_tag_from_artifact_uri(model_artifact)
         model_parameters["artifact_uri"] = model_parameters.get(
             "artifact_uri", model_artifact
         )
@@ -2923,7 +2991,7 @@ def params_to_step(
         step = QueueStep(name, **class_args)
 
     elif class_name and hasattr(class_name, "to_dict"):
-        struct = class_name.to_dict()
+        struct = deepcopy(class_name.to_dict())
         kind = struct.get("kind", StepKinds.task)
         name = (
             name