mlrun 1.10.0rc15__py3-none-any.whl → 1.10.0rc16__py3-none-any.whl

mlrun/artifacts/llm_prompt.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 import json
 import tempfile
+from collections import defaultdict
 from typing import Optional, Union
 
 import mlrun
@@ -253,3 +254,8 @@ class LLMPromptArtifact(Artifact):
             self.spec.prompt_template = None
             self._src_is_temp = True
         super().before_log()
+
+
+class PlaceholderDefaultDict(defaultdict):
+    def __missing__(self, key):
+        return f"{{{key}}}"

mlrun/common/constants.py

@@ -83,6 +83,7 @@ class MLRunInternalLabels:
     mlrun_type = "mlrun__type"
     original_workflow_id = "original-workflow-id"
     workflow_id = "workflow-id"
+    retrying = "retrying"
 
     owner = "owner"
     v3io_user = "v3io_user"

mlrun/common/schemas/workflow.py

@@ -51,6 +51,8 @@ class RerunWorkflowRequest(pydantic.v1.BaseModel):
     run_id: typing.Optional[str] = None
     notifications: typing.Optional[list[Notification]] = None
     workflow_runner_node_selector: typing.Optional[dict[str, str]] = None
+    original_workflow_runner_uid: typing.Optional[str] = None
+    original_workflow_name: typing.Optional[str] = None
 
 
 class WorkflowResponse(pydantic.v1.BaseModel):

mlrun/db/base.py

@@ -55,6 +55,12 @@ class RunDBInterface(ABC):
     def update_run(self, updates: dict, uid, project="", iter=0):
         pass
 
+    @abstractmethod
+    def set_run_retrying_status(
+        self, project: str, name: str, run_id: str, retrying: bool
+    ):
+        pass
+
     @abstractmethod
     def abort_run(self, uid, project="", iter=0, timeout=45, status_text=""):
         pass

mlrun/db/httpdb.py

@@ -4741,6 +4741,28 @@ class HTTPRunDB(RunDBInterface):
         )
         return mlrun.common.schemas.GetWorkflowResponse(**response.json())
 
+    def set_run_retrying_status(
+        self, project: str, name: str, run_id: str, retrying: bool = False
+    ):
+        """
+        Toggle the “retrying” label on a workflow-runner run.
+
+        This will POST to the workflows endpoint to either add or remove the
+        `retrying` flag on a specific run, which prevents parallel retries.
+
+        :param project:   The project name under which the workflow is defined.
+        :param name:      The workflow name (as in the URL path).
+        :param run_id:    The UID of the workflow-runner run to update.
+        :param retrying:  True to add the `retrying` label, False to remove it.
+
+        :raises MLRunHTTPError: If the HTTP request fails or returns an error status.
+        """
+        path = f"projects/{project}/workflows/{name}/runs/{run_id}/set-retry-status"
+        params = {"retrying": retrying}
+        self.api_call(
+            "POST", path, f"set retrying on {project}/{run_id}", params=params
+        )
+
     def load_project(
         self,
         name: str,

mlrun/db/nopdb.py

@@ -72,6 +72,11 @@ class NopDB(RunDBInterface):
     def update_run(self, updates: dict, uid, project="", iter=0):
         pass
 
+    def set_run_retrying_status(
+        self, project: str, name: str, run_id: str, retrying: bool
+    ):
+        pass
+
     def abort_run(self, uid, project="", iter=0, timeout=45, status_text=""):
         pass
 

mlrun/launcher/local.py

@@ -276,6 +276,19 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
                     args = sp[1:]
         return command, args
 
+    def _validate_run(
+        self,
+        runtime: "mlrun.runtimes.BaseRuntime",
+        run: "mlrun.run.RunObject",
+    ):
+        super()._validate_run(runtime, run)
+        if self._is_run_local and run.spec.retry.count:
+            logger.warning(
+                "Retry is not supported for local runs, ignoring retry settings",
+                retry=run.spec.retry.to_dict(),
+            )
+            run.spec.retry.count = 0
+
     def _push_notifications(
         self, runobj: "mlrun.run.RunObject", runtime: "mlrun.runtimes.BaseRuntime"
     ):

mlrun/projects/pipelines.py

@@ -1072,7 +1072,11 @@ def github_webhook(request):
 
 
 def rerun_workflow(
-    context: mlrun.execution.MLClientCtx, run_uid: str, project_name: str
+    context: mlrun.execution.MLClientCtx,
+    run_uid: str,
+    project_name: str,
+    original_runner_uid: str,
+    original_workflow_name: str,
 ):
     """
     Re-run a workflow by retrying a previously failed KFP pipeline.
@@ -1080,8 +1084,11 @@ def rerun_workflow(
     :param context:      MLRun context.
     :param run_uid:      The run UID of the original workflow to retry.
     :param project_name: The project name.
+    :param original_runner_uid: The original workflow runner UID.
+    :param original_workflow_name: The original workflow name.
     """
     db = mlrun.get_run_db()
+    new_pipeline_id = None
 
     try:
         # Invoke the KFP retry endpoint (direct-submit mode)
@@ -1096,6 +1103,24 @@ def rerun_workflow(
             rerun_of_workflow=run_uid,
         )
 
+        # Enqueue "running" notifications server-side for this RerunRunner run
+        db.push_run_notifications(context.uid, project_name)
+
+        context.set_label(
+            mlrun_constants.MLRunInternalLabels.workflow_id, new_pipeline_id
+        )
+        context.update_run()
+
+        context.log_result("workflow_id", new_pipeline_id)
+
+        pipeline = wait_for_pipeline_completion(
+            new_pipeline_id,
+            project=project_name,
+        )
+
+        final_state = pipeline["run"]["status"]
+        context.log_result("workflow_state", final_state, commit=True)
+
     except mlrun.errors.MLRunHTTPError as http_exc:
         logger.error(
             "Failed calling KFP retry API",
@@ -1104,33 +1129,28 @@ def rerun_workflow(
         )
         raise
 
-    # Enqueue "running" notifications server-side for this RerunRunner run
-    db.push_run_notifications(context.uid, project_name)
-
-    context.set_label(mlrun_constants.MLRunInternalLabels.workflow_id, new_pipeline_id)
-    context.update_run()
-
-    context.log_result("workflow_id", new_pipeline_id)
-
-    try:
-        pipeline = wait_for_pipeline_completion(
-            new_pipeline_id,
-            project=project_name,
-        )
     except Exception as exc:
-        mlrun.utils.logger.error(
-            "Failed waiting for workflow completion",
+        logger.error(
+            "Error during rerun_workflow execution",
+            error=err_to_str(exc),
             rerun_pipeline_id=new_pipeline_id,
-            exc=err_to_str(exc),
         )
-    else:
-        final_state = pipeline["run"]["status"]
-        context.log_result("workflow_state", final_state, commit=True)
+        raise
 
-        if final_state != mlrun_pipelines.common.models.RunStatuses.succeeded:
-            raise mlrun.errors.MLRunRuntimeError(
-                f"Pipeline retry of {run_uid} finished in state={final_state}"
-            )
+    finally:
+        # Once the rerun has finished, clear the “retrying” label on the original runner
+        # so that subsequent retry requests can acquire the lock again.
+        db.set_run_retrying_status(
+            project=project_name,
+            name=original_workflow_name,
+            run_id=original_runner_uid,
+            retrying=False,
+        )
+
+    if final_state != mlrun_pipelines.common.models.RunStatuses.succeeded:
+        raise mlrun.errors.MLRunRuntimeError(
+            f"Pipeline retry of {run_uid} finished in state={final_state}"
+        )
 
 
 def load_and_run(context, *args, **kwargs):

mlrun/projects/project.py

@@ -5073,7 +5073,6 @@ class MlrunProject(ModelObj):
         :param states: List only runs whose state is one of the provided states.
         :param sort: Whether to sort the result according to their start time. Otherwise, results will be
             returned by their internal order in the DB (order will not be guaranteed).
-        :param last: Deprecated - currently not used (will be removed in 1.10.0).
         :param iter: If ``True`` return runs from all iterations. Otherwise, return only runs whose ``iter`` is 0.
         :param start_time_from: Filter by run start time in ``[start_time_from, start_time_to]``.
         :param start_time_to: Filter by run start time in ``[start_time_from, start_time_to]``.

mlrun/runtimes/utils.py

@@ -445,8 +445,6 @@ def enrich_run_labels(
     labels_enrichment = {
         mlrun_constants.MLRunInternalLabels.owner: os.environ.get("V3IO_USERNAME")
         or getpass.getuser(),
-        # TODO: remove this in 1.10.0
-        mlrun_constants.MLRunInternalLabels.v3io_user: os.environ.get("V3IO_USERNAME"),
     }
 
     # Resolve which label keys to enrich

mlrun/serving/server.py

@@ -350,33 +350,33 @@ def add_error_raiser_step(
     monitored_steps_raisers = {}
     user_steps = list(graph.steps.values())
     for monitored_step in monitored_steps.values():
-        if monitored_step.raise_exception:
-            error_step = graph.add_step(
-                class_name="mlrun.serving.states.ModelRunnerErrorRaiser",
-                name=f"{monitored_step.name}_error_raise",
-                after=monitored_step.name,
-                full_event=True,
-                raise_exception=monitored_step.raise_exception,
-                models_names=list(monitored_step.class_args["models"].keys()),
-                model_endpoint_creation_strategy=mlrun.common.schemas.ModelEndpointCreationStrategy.SKIP,
-            )
-            if monitored_step.responder:
-                monitored_step.responder = False
-                error_step.respond()
-            monitored_steps_raisers[monitored_step.name] = error_step.name
-            error_step.on_error = monitored_step.on_error
-    for step in user_steps:
-        if step.after:
-            if isinstance(step.after, list):
-                for i in range(len(step.after)):
-                    if step.after[i] in monitored_steps_raisers:
-                        step.after[i] = monitored_steps_raisers[step.after[i]]
-            else:
-                if (
-                    isinstance(step.after, str)
-                    and step.after in monitored_steps_raisers
-                ):
-                    step.after = monitored_steps_raisers[step.after]
+        error_step = graph.add_step(
+            class_name="mlrun.serving.states.ModelRunnerErrorRaiser",
+            name=f"{monitored_step.name}_error_raise",
+            after=monitored_step.name,
+            full_event=True,
+            raise_exception=monitored_step.raise_exception,
+            models_names=list(monitored_step.class_args["models"].keys()),
+            model_endpoint_creation_strategy=mlrun.common.schemas.ModelEndpointCreationStrategy.SKIP,
+        )
+        if monitored_step.responder:
+            monitored_step.responder = False
+            error_step.respond()
+        monitored_steps_raisers[monitored_step.name] = error_step.name
+        error_step.on_error = monitored_step.on_error
+    if monitored_steps_raisers:
+        for step in user_steps:
+            if step.after:
+                if isinstance(step.after, list):
+                    for i in range(len(step.after)):
+                        if step.after[i] in monitored_steps_raisers:
+                            step.after[i] = monitored_steps_raisers[step.after[i]]
+                else:
+                    if (
+                        isinstance(step.after, str)
+                        and step.after in monitored_steps_raisers
+                    ):
+                        step.after = monitored_steps_raisers[step.after]
     return graph
 
 

mlrun/serving/states.py

@@ -35,7 +35,7 @@ from storey import ParallelExecutionMechanisms
 import mlrun
 import mlrun.artifacts
 import mlrun.common.schemas as schemas
-from mlrun.artifacts.llm_prompt import LLMPromptArtifact
+from mlrun.artifacts.llm_prompt import LLMPromptArtifact, PlaceholderDefaultDict
 from mlrun.artifacts.model import ModelArtifact
 from mlrun.datastore.datastore_profile import (
     DatastoreProfileKafkaSource,
@@ -45,7 +45,7 @@ from mlrun.datastore.datastore_profile import (
 )
 from mlrun.datastore.model_provider.model_provider import ModelProvider
 from mlrun.datastore.storeytargets import KafkaStoreyTarget, StreamStoreyTarget
-from mlrun.utils import logger
+from mlrun.utils import get_data_from_path, logger, split_path
 
 from ..config import config
 from ..datastore import get_stream_pusher
@@ -501,10 +501,15 @@ class BaseStep(ModelObj):
     def verify_model_runner_step(
         self,
         step: "ModelRunnerStep",
+        step_model_endpoints_names: Optional[list[str]] = None,
+        verify_shared_models: bool = True,
     ):
         """
         Verify ModelRunnerStep, can be part of Flow graph and models can not repeat in graph.
-        :param step: ModelRunnerStep to verify
+        :param step:                        ModelRunnerStep to verify
+        :param step_model_endpoints_names:  List of model endpoints names that are in the step.
+                                            if provided will ignore step models and verify only the models on list.
+        :param verify_shared_models:        If True, verify that shared models are defined in the graph.
         """
 
         if not isinstance(step, ModelRunnerStep):
@@ -516,7 +521,7 @@ class BaseStep(ModelObj):
             raise GraphError(
                 "ModelRunnerStep can be added to 'Flow' topology graph only"
             )
-        step_model_endpoints_names = list(
+        step_model_endpoints_names = step_model_endpoints_names or list(
             step.class_args.get(schemas.ModelRunnerStepData.MODELS, {}).keys()
         )
         # Get all model_endpoints names that are in both lists
@@ -530,8 +535,9 @@ class BaseStep(ModelObj):
                 f"The graph already contains the model endpoints named - {common_endpoints_names}."
             )
 
-        # Check if shared models are defined in the graph
-        self._verify_shared_models(root, step, step_model_endpoints_names)
+        if verify_shared_models:
+            # Check if shared models are defined in the graph
+            self._verify_shared_models(root, step, step_model_endpoints_names)
         # Update model endpoints names in the root step
         root.update_model_endpoints_names(step_model_endpoints_names)
 
@@ -569,7 +575,9 @@ class BaseStep(ModelObj):
                 llm_artifact, _ = mlrun.store_manager.get_store_artifact(
                     model_artifact_uri
                 )
-                model_artifact_uri = llm_artifact.spec.parent_uri
+                model_artifact_uri = mlrun.utils.remove_tag_from_artifact_uri(
+                    llm_artifact.spec.parent_uri
+                )
             actual_shared_name = root.get_shared_model_name_by_artifact_uri(
                 model_artifact_uri
             )
@@ -1148,11 +1156,11 @@ class Model(storey.ParallelExecutionRunnable, ModelObj):
     def init(self):
         self.load()
 
-    def predict(self, body: Any) -> Any:
+    def predict(self, body: Any, **kwargs) -> Any:
         """Override to implement prediction logic. If the logic requires asyncio, override predict_async() instead."""
         return body
 
-    async def predict_async(self, body: Any) -> Any:
+    async def predict_async(self, body: Any, **kwargs) -> Any:
         """Override to implement prediction logic if the logic requires asyncio."""
         return body
 
@@ -1197,11 +1205,18 @@ class Model(storey.ParallelExecutionRunnable, ModelObj):
 
 
 class LLModel(Model):
-    def __init__(self, name: str, **kwargs):
+    def __init__(
+        self, name: str, input_path: Optional[Union[str, list[str]]], **kwargs
+    ):
         super().__init__(name, **kwargs)
+        self._input_path = split_path(input_path)
 
     def predict(
-        self, body: Any, messages: list[dict], model_configuration: dict
+        self,
+        body: Any,
+        messages: Optional[list[dict]] = None,
+        model_configuration: Optional[dict] = None,
+        **kwargs,
     ) -> Any:
         if isinstance(
             self.invocation_artifact, mlrun.artifacts.LLMPromptArtifact
@@ -1214,7 +1229,11 @@ class LLModel(Model):
         return body
 
     async def predict_async(
-        self, body: Any, messages: list[dict], model_configuration: dict
+        self,
+        body: Any,
+        messages: Optional[list[dict]] = None,
+        model_configuration: Optional[dict] = None,
+        **kwargs,
     ) -> Any:
         if isinstance(
             self.invocation_artifact, mlrun.artifacts.LLMPromptArtifact
@@ -1262,12 +1281,34 @@ class LLModel(Model):
             return None, None
         prompt_legend = llm_prompt_artifact.spec.prompt_legend
         prompt_template = deepcopy(llm_prompt_artifact.read_prompt())
-        kwargs = {
-            place_holder: body.get(body_map["field"])
-            for place_holder, body_map in prompt_legend.items()
-        }
-        for d in prompt_template:
-            d["content"] = d["content"].format(**kwargs)
+        input_data = copy(get_data_from_path(self._input_path, body))
+        if isinstance(input_data, dict):
+            kwargs = (
+                {
+                    place_holder: input_data.get(body_map["field"])
+                    for place_holder, body_map in prompt_legend.items()
+                }
+                if prompt_legend
+                else {}
+            )
+            input_data.update(kwargs)
+            default_place_holders = PlaceholderDefaultDict(lambda: None, input_data)
+            for message in prompt_template:
+                try:
+                    message["content"] = message["content"].format(**input_data)
+                except KeyError as e:
+                    logger.warning(
+                        "Input data was missing a placeholder, placeholder stay unformatted",
+                        key_error=e,
+                    )
+                    message["content"] = message["content"].format_map(
+                        default_place_holders
+                    )
+        else:
+            logger.warning(
+                f"Expected input data to be a dict, but received input data from type {type(input_data)} prompt "
+                f"template stay unformatted",
+            )
         return prompt_template, llm_prompt_artifact.spec.model_configuration
 
 
@@ -1567,11 +1608,27 @@ class ModelRunnerStep(MonitoredStep):
           :param outputs:             list of the model outputs (e.g. labels) ,if provided will override the outputs
                                       that been configured in the model artifact, please note that those outputs need to
                                       be equal to the model_class predict method outputs (length, and order)
-          :param input_path:          input path inside the user event, expect scopes to be defined by dot notation
-                                      (e.g "inputs.my_model_inputs"). expects list or dictionary type object in path.
-          :param result_path:         result path inside the user output event, expect scopes to be defined by dot
-                                      notation (e.g "outputs.my_model_outputs") expects list or dictionary type object
-                                      in path.
+          :param input_path:          when specified selects the key/path in the event to use as model monitoring inputs
+                                      this require that the event body will behave like a dict, expects scopes to be
+                                      defined by dot notation (e.g "data.d").
+                                      examples: input_path="data.b"
+                                      event: {"data":{"a": 5, "b": 7}}, means monitored body will be 7.
+                                      event: {"data":{"a": [5, 9], "b": [7, 8]}} means monitored body will be [7,8].
+                                      event: {"data":{"a": "extra_data", "b": {"f0": [1, 2]}}} means monitored body will
+                                      be {"f0": [1, 2]}.
+                                      if a ``list`` or ``list of lists`` is provided, it must follow the order and
+                                      size defined by the input schema.
+          :param result_path:         when specified selects the key/path in the output event to use as model monitoring
+                                      outputs this require that the output event body will behave like a dict,
+                                      expects scopes to be defined by dot notation (e.g "data.d").
+                                      examples: result_path="out.b"
+                                      event: {"out":{"a": 5, "b": 7}}, means monitored body will be 7.
+                                      event: {"out":{"a": [5, 9], "b": [7, 8]}} means monitored body will be [7,8]
+                                      event: {"out":{"a": "extra_data", "b": {"f0": [1, 2]}}} means monitored body will
+                                      be {"f0": [1, 2]}
+                                      if a ``list`` or ``list of lists`` is provided, it must follow the order and
+                                      size defined by the output schema.
+
           :param override:            bool allow override existing model on the current ModelRunnerStep.
           :param model_parameters:    Parameters for model instantiation
         """
@@ -1590,7 +1647,7 @@ class ModelRunnerStep(MonitoredStep):
         ):
             try:
                 model_artifact, _ = mlrun.store_manager.get_store_artifact(
-                    model_artifact
+                    mlrun.utils.remove_tag_from_artifact_uri(model_artifact)
                 )
             except mlrun.errors.MLRunNotFoundError:
                 raise mlrun.errors.MLRunInvalidArgumentError("Artifact not found.")
@@ -1602,6 +1659,11 @@ class ModelRunnerStep(MonitoredStep):
             if isinstance(model_artifact, mlrun.artifacts.Artifact)
             else model_artifact
         )
+        model_artifact = (
+            mlrun.utils.remove_tag_from_artifact_uri(model_artifact)
+            if model_artifact
+            else None
+        )
         model_parameters["artifact_uri"] = model_parameters.get(
             "artifact_uri", model_artifact
         )
@@ -1617,6 +1679,11 @@ class ModelRunnerStep(MonitoredStep):
             raise mlrun.errors.MLRunInvalidArgumentError(
                 f"Model with name {endpoint_name} already exists in this ModelRunnerStep."
             )
+        root = self._extract_root_step()
+        if isinstance(root, RootFlowStep):
+            self.verify_model_runner_step(
+                self, [endpoint_name], verify_shared_models=False
+            )
         ParallelExecutionMechanisms.validate(execution_mechanism)
         self.class_args[schemas.ModelRunnerStepData.MODEL_TO_EXECUTION_MECHANISM] = (
             self.class_args.get(
@@ -1693,15 +1760,6 @@ class ModelRunnerStep(MonitoredStep):
             )
         return output_schema
 
-    @staticmethod
-    def _split_path(path: str) -> Union[str, list[str], None]:
-        if path is not None:
-            parsed_path = path.split(".")
-            if len(parsed_path) == 1:
-                parsed_path = parsed_path[0]
-            return parsed_path
-        return path
-
     def _calculate_monitoring_data(self) -> dict[str, dict[str, str]]:
         monitoring_data = deepcopy(
             self.class_args.get(
@@ -1726,15 +1784,11 @@ class ModelRunnerStep(MonitoredStep):
                 ][model][schemas.MonitoringData.OUTPUTS] = monitoring_data[model][
                     schemas.MonitoringData.OUTPUTS
                 ]
-                monitoring_data[model][schemas.MonitoringData.INPUT_PATH] = (
-                    self._split_path(
-                        monitoring_data[model][schemas.MonitoringData.INPUT_PATH]
-                    )
+                monitoring_data[model][schemas.MonitoringData.INPUT_PATH] = split_path(
+                    monitoring_data[model][schemas.MonitoringData.INPUT_PATH]
                 )
-                monitoring_data[model][schemas.MonitoringData.RESULT_PATH] = (
-                    self._split_path(
-                        monitoring_data[model][schemas.MonitoringData.RESULT_PATH]
-                    )
+                monitoring_data[model][schemas.MonitoringData.RESULT_PATH] = split_path(
+                    monitoring_data[model][schemas.MonitoringData.RESULT_PATH]
                 )
             return monitoring_data
 
@@ -1752,6 +1806,13 @@ class ModelRunnerStep(MonitoredStep):
             model_selector = get_class(model_selector, namespace)()
         model_objects = []
         for model, model_params in models.values():
+            model_params[schemas.MonitoringData.INPUT_PATH] = (
+                self.class_args.get(
+                    mlrun.common.schemas.ModelRunnerStepData.MONITORING_DATA, {}
+                )
+                .get(model_params.get("name"), {})
+                .get(schemas.MonitoringData.INPUT_PATH)
+            )
             model = get_class(model, namespace).from_dict(
                 model_params, init_with_params=True
             )
@@ -2401,7 +2462,13 @@ class FlowStep(BaseStep):
         if not step.before and not any(
             [step.name in other_step.after for other_step in self._steps.values()]
         ):
-            step.responder = True
+            if any(
+                [
+                    getattr(step_in_graph, "responder", False)
+                    for step_in_graph in self._steps.values()
+                ]
+            ):
+                step.responder = True
             return
 
         for step_name in step.before:
@@ -2484,7 +2551,7 @@ class RootFlowStep(FlowStep):
         name: str,
         model_class: Union[str, Model],
         execution_mechanism: Union[str, ParallelExecutionMechanisms],
-        model_artifact: Optional[Union[str, ModelArtifact]],
+        model_artifact: Union[str, ModelArtifact],
         override: bool = False,
         **model_parameters,
     ) -> None:
@@ -2536,6 +2603,7 @@ class RootFlowStep(FlowStep):
             if isinstance(model_artifact, mlrun.artifacts.Artifact)
             else model_artifact
         )
+        model_artifact = mlrun.utils.remove_tag_from_artifact_uri(model_artifact)
         model_parameters["artifact_uri"] = model_parameters.get(
             "artifact_uri", model_artifact
         )
@@ -2923,7 +2991,7 @@ def params_to_step(
         step = QueueStep(name, **class_args)
 
     elif class_name and hasattr(class_name, "to_dict"):
-        struct = class_name.to_dict()
+        struct = deepcopy(class_name.to_dict())
         kind = struct.get("kind", StepKinds.task)
         name = (
             name

mlrun/serving/system_steps.py

@@ -13,10 +13,10 @@
 # limitations under the License.
 
 import random
-from copy import deepcopy
 from datetime import timedelta
 from typing import Any, Optional, Union
 
+import numpy as np
 import storey
 
 import mlrun
@@ -24,7 +24,7 @@ import mlrun.artifacts
 import mlrun.common.schemas.model_monitoring as mm_schemas
 import mlrun.serving
 from mlrun.common.schemas import MonitoringData
-from mlrun.utils import logger
+from mlrun.utils import get_data_from_path, logger
 
 
 class MonitoringPreProcessor(storey.MapClass):
@@ -45,24 +45,13 @@ class MonitoringPreProcessor(storey.MapClass):
         result_path = model_monitoring_data.get(MonitoringData.RESULT_PATH)
         input_path = model_monitoring_data.get(MonitoringData.INPUT_PATH)
 
-        result = self._get_data_from_path(
-            result_path, event.body.get(model, event.body)
-        )
+        result = get_data_from_path(result_path, event.body.get(model, event.body))
         output_schema = model_monitoring_data.get(MonitoringData.OUTPUTS)
         input_schema = model_monitoring_data.get(MonitoringData.INPUTS)
         logger.debug("output schema retrieved", output_schema=output_schema)
         if isinstance(result, dict):
-            if len(result) > 1:
-                # transpose by key the outputs:
-                outputs = self.transpose_by_key(result, output_schema)
-            elif len(result) == 1:
-                outputs = (
-                    result[output_schema[0]]
-                    if output_schema
-                    else list(result.values())[0]
-                )
-            else:
-                outputs = []
+            # transpose by key the outputs:
+            outputs = self.transpose_by_key(result, output_schema)
             if not output_schema:
                 logger.warn(
                     "Output schema was not provided using Project:log_model or by ModelRunnerStep:add_model order "
@@ -72,16 +61,14 @@ class MonitoringPreProcessor(storey.MapClass):
             outputs = result
 
         event_inputs = event._metadata.get("inputs", {})
-        event_inputs = self._get_data_from_path(input_path, event_inputs)
+        event_inputs = get_data_from_path(input_path, event_inputs)
         if isinstance(event_inputs, dict):
-            if len(event_inputs) > 1:
-                # transpose by key the inputs:
-                inputs = self.transpose_by_key(event_inputs, input_schema)
-            else:
-                inputs = (
-                    event_inputs[input_schema[0]]
-                    if input_schema
-                    else list(result.values())[0]
+            # transpose by key the inputs:
+            inputs = self.transpose_by_key(event_inputs, input_schema)
+            if not input_schema:
+                logger.warn(
+                    "Input schema was not provided using by ModelRunnerStep:add_model, order "
+                    "may not preserved"
                 )
         else:
             inputs = event_inputs
@@ -104,6 +91,11 @@ class MonitoringPreProcessor(storey.MapClass):
                     output_len=len(outputs),
                     schema_len=len(output_schema),
                 )
+            if len(inputs) != len(outputs):
+                logger.warn(
+                    "outputs and inputs are not in the same length check 'input_path' and "
+                    "'output_path' was specified if needed"
+                )
         request = {"inputs": inputs, "id": getattr(event, "id", None)}
         resp = {"outputs": outputs}
 
@@ -111,41 +103,73 @@ class MonitoringPreProcessor(storey.MapClass):
 
     @staticmethod
     def transpose_by_key(
-        data_to_transpose, schema: Optional[list[str]] = None
-    ) -> list[list[float]]:
-        values = (
-            list(data_to_transpose.values())
-            if not schema
-            else [data_to_transpose[key] for key in schema]
-        )
-        if values and not isinstance(values[0], list):
-            values = [values]
-        transposed = (
-            list(map(list, zip(*values)))
-            if all(isinstance(v, list) for v in values) and len(values) > 1
-            else values
-        )
-        return transposed
+        data: dict, schema: Optional[Union[str, list[str]]] = None
+    ) -> Union[list[float], list[list[float]]]:
+        """
+        Transpose values from a dictionary by keys.
 
-    @staticmethod
-    def _get_data_from_path(
-        path: Union[str, list[str], None], data: dict
-    ) -> dict[str, Any]:
-        if isinstance(path, str):
-            output_data = data.get(path)
-        elif isinstance(path, list):
-            output_data = deepcopy(data)
-            for key in path:
-                output_data = output_data.get(key, {})
-        elif path is None:
-            output_data = data
+        Given a dictionary and an optional schema (a key or list of keys), this function:
+        - Extracts the values for the specified keys (or all keys if no schema is provided).
+        - Ensures the data is represented as a list of rows, then transposes it (i.e., switches rows to columns).
+        - Handles edge cases:
+            * If a single scalar or single-element list is provided, returns a flat list.
+            * If a single key is provided (as a string or a list with one element), handles it properly.
+            * If only one row with len of one remains after transposition, unwraps it to avoid nested list-of-one.
+
+        Example::
+
+            transpose_by_key({"a": 1})
+            # returns: [1]
+
+            transpose_by_key({"a": [1, 2]})
+            # returns: [1 ,2]
+
+            transpose_by_key({"a": [1, 2], "b": [3, 4]})
+            # returns: [[1, 3], [2, 4]]
+
+        :param data:     Dictionary with values that are either scalars or lists.
+        :param schema:   Optional key or list of keys to extract. If not provided, all keys are used.
+                         Can be a string (single key) or a list of strings.
+
+        :return:         Transposed values:
+                         * If result is a single column or row, returns a flat list.
+                         * If result is a matrix, returns a list of lists.
+
+        :raises ValueError: If the values include a mix of scalars and lists, or if the list lengths do not match.
+        """
+
+        # Normalize schema to list
+        if not schema:
+            keys = list(data.keys())
+        elif isinstance(schema, str):
+            keys = [schema]
         else:
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "Expected path be of type str or list of str or None"
+            keys = schema
+
+        values = [data[key] for key in keys]
+
+        # Detect if all are scalars ie: int,float,str
+        all_scalars = all(not isinstance(v, (list, tuple, np.ndarray)) for v in values)
+        all_lists = all(isinstance(v, (list, tuple, np.ndarray)) for v in values)
+
+        if not (all_scalars or all_lists):
+            raise ValueError(
+                "All values must be either scalars or lists of equal length."
             )
-        if isinstance(output_data, (int, float)):
-            output_data = [output_data]
-        return output_data
+
+        if all_scalars:
+            transposed = np.array([values])
+        elif all_lists and len(keys) > 1:
+            arrays = [np.array(v) for v in values]
+            mat = np.stack(arrays, axis=0)
+            transposed = mat.T
+        else:
+            return values[0]
+
+        if transposed.shape[1] == 1 and transposed.shape[0] == 1:
+            # Transform [[0]] -> [0]:
+            return transposed[:, 0].tolist()
+        return transposed.tolist()
 
     def do(self, event):
         monitoring_event_list = []
@@ -337,7 +361,9 @@ class SamplingStep(storey.MapClass):
             event=event,
             sampling_percentage=self.sampling_percentage,
         )
-        if self.sampling_percentage != 100:
+        if self.sampling_percentage != 100 and not event.get(
+            mm_schemas.StreamProcessingEvent.ERROR
+        ):
             request = event[mm_schemas.StreamProcessingEvent.REQUEST]
             num_of_inputs = len(request["inputs"])
             sampled_requests_indices = self._pick_random_requests(

mlrun/utils/helpers.py

@@ -29,6 +29,7 @@ import traceback
 import typing
 import uuid
 import warnings
+from copy import deepcopy
 from datetime import datetime, timedelta, timezone
 from importlib import import_module, reload
 from os import path
@@ -786,6 +787,22 @@ def generate_artifact_uri(
     return artifact_uri
 
 
+def remove_tag_from_artifact_uri(uri: str) -> Optional[str]:
+    """
+    Remove the `:<tag>` part from a URI with pattern:
+    [store://][<project>/]<key>[#<iter>][:<tag>][@<tree>][^<uid>]
+
+    Returns the URI without the tag section.
+
+    Examples:
+        "store://proj/key:latest" => "store://proj/key"
+        "key#1:dev@tree^uid" => "key#1@tree^uid"
+        "store://key:tag" => "store://key"
+        "store://models/remote-model-project/my_model#0@tree" => unchanged (no tag)
+    """
+    return re.sub(r"(?<=/[^/:]\+):[^@^:\s]+(?=(@|\^|$))", "", uri)
+
+
 def extend_hub_uri_if_needed(uri) -> tuple[str, bool]:
     """
     Retrieve the full uri of the item's yaml in the hub.
@@ -2376,3 +2393,32 @@ def encode_user_code(
             "Consider using `with_source_archive` to add user code as a remote source to the function."
         )
     return encoded
+
+
+def split_path(path: str) -> typing.Union[str, list[str], None]:
+    if path is not None:
+        parsed_path = path.split(".")
+        if len(parsed_path) == 1:
+            parsed_path = parsed_path[0]
+        return parsed_path
+    return path
+
+
+def get_data_from_path(
+    path: typing.Union[str, list[str], None], data: dict
+) -> dict[str, Any]:
+    if isinstance(path, str):
+        output_data = data.get(path)
+    elif isinstance(path, list):
+        output_data = deepcopy(data)
+        for key in path:
+            output_data = output_data.get(key, {})
+    elif path is None:
+        output_data = data
+    else:
+        raise mlrun.errors.MLRunInvalidArgumentError(
+            "Expected path be of type str or list of str or None"
+        )
+    if isinstance(output_data, (int, float)):
+        output_data = [output_data]
+    return output_data

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "ab4eba87d14deb67eed3b45e18965920951d7993",
-  "version": "1.10.0-rc15"
+  "git_commit": "78045e1e85e7c81eee93682240c4ebe7b22fa67c",
+  "version": "1.10.0-rc16"
 }

{mlrun-1.10.0rc15.dist-info → mlrun-1.10.0rc16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.10.0rc15
+Version: 1.10.0rc16
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv

{mlrun-1.10.0rc15.dist-info → mlrun-1.10.0rc16.dist-info}/RECORD

@@ -18,12 +18,12 @@ mlrun/artifacts/base.py,sha256=6x_2KPMNOciiNNUsiKgJ-b6ejxAHm_Ro22xODLoTc44,28559
 mlrun/artifacts/dataset.py,sha256=bhb5Kfbs8P28yjnpN76th5lLEUl5nAqD4VqVzHEVPrM,16421
 mlrun/artifacts/document.py,sha256=p5HsWdmIIJ0NahS7y3EEQN2tfHtUrUmUG-8BEEyi_Jc,17373
 mlrun/artifacts/helpers.py,sha256=ejTEC9vkI2w5FHn5Gopw3VEIxuni0bazWUnR6BBWZfU,1662
-mlrun/artifacts/llm_prompt.py,sha256=uP_uq-SpbVs9uV9fFG3yF9e_X4XuXYt_EHAu4feaBfQ,9414
+mlrun/artifacts/llm_prompt.py,sha256=mwImD34RAXp2lC5YEUfjD_iwuQr705ZjOHbDtMh4JVI,9555
 mlrun/artifacts/manager.py,sha256=_cDNCS7wwmFIsucJ2uOgHxZQECmIGb8Wye64b6oLgKU,16642
 mlrun/artifacts/model.py,sha256=8EVaD70SOkTohQIWqkDk0MEwskdofxs3wJTgspa2sho,25615
 mlrun/artifacts/plots.py,sha256=wmaxVXiAPSCyn3M7pIlcBu9pP3O8lrq0Ewx6iHRDF9s,4238
 mlrun/common/__init__.py,sha256=kXGBqhLN0rlAx0kTXhozGzFsIdSqW0uTSKMmsLgq_is,569
-mlrun/common/constants.py,sha256=tWqaog0fe3ZU6sIGToB8Joo7AY_3QjpnxA2GkiiAtj8,4033
+mlrun/common/constants.py,sha256=bkWWccWZJHB08mkSrrnWuCFXZWdNKf4Q9SwuLowQrVM,4059
 mlrun/common/helpers.py,sha256=DIdqs_eN3gO5bZ8iFobIvx8cEiOxYxhFIyut6-O69T0,1385
 mlrun/common/secrets.py,sha256=8g9xtIw-9DGcwiZRT62a5ozSQM-aYo8yK5Ghey9WM0g,5179
 mlrun/common/types.py,sha256=1gxThbmC0Vd0U1ffIkEwz4T4S7JOgHt70rvw8TCO21c,1073
@@ -73,7 +73,7 @@ mlrun/common/schemas/schedule.py,sha256=L7z9Lp06-xmFmdp0q5PypCU_DCl6zZIyQTVoJa01
 mlrun/common/schemas/secret.py,sha256=Td2UAeWHSAdA4nIP3rQv_PIVKVqcBnCnK6xjr528tS8,1486
 mlrun/common/schemas/serving.py,sha256=-3U45YLtmVWMZrx4R8kaPgFGoJ4JmD7RE3nydpYNTz8,1359
 mlrun/common/schemas/tag.py,sha256=1wqEiAujsElojWb3qmuyfcaLFjXSNAAQdafkDx7fkn0,891
-mlrun/common/schemas/workflow.py,sha256=4KeTUIZCkIgEIKNDbMeJqyhUmIKvLdX1bQSNsmYMCwg,2378
+mlrun/common/schemas/workflow.py,sha256=GS6RNp66v7iV2flBMXZs4R5mQr15KJx6PMXFlAjyOmc,2496
 mlrun/common/schemas/model_monitoring/__init__.py,sha256=lQkWiDZagEmZd7pNE_-ySVJEzTjEzH-JS6OKZPmJiVk,1907
 mlrun/common/schemas/model_monitoring/constants.py,sha256=yjTaSGiRs0zYIE20QSuJuMNnS5iuJpnV1wBiq7leVpg,13238
 mlrun/common/schemas/model_monitoring/functions.py,sha256=GpfSGp05D87wEKemECD3USL368pvnAM2WfS-nef5qOg,2210
@@ -115,10 +115,10 @@ mlrun/datastore/wasbfs/__init__.py,sha256=s5Ul-0kAhYqFjKDR2X0O2vDGDbLQQduElb32Ev
 mlrun/datastore/wasbfs/fs.py,sha256=ge8NK__5vTcFT-krI155_8RDUywQw4SIRX6BWATXy9Q,6299
 mlrun/db/__init__.py,sha256=WqJ4x8lqJ7ZoKbhEyFqkYADd9P6E3citckx9e9ZLcIU,1163
 mlrun/db/auth_utils.py,sha256=hpg8D2r82oN0BWabuWN04BTNZ7jYMAF242YSUpK7LFM,5211
-mlrun/db/base.py,sha256=NZE8LoDOmwPCr1LGY9hX_2knpNVJluS0QZghi-m49YQ,31903
+mlrun/db/base.py,sha256=afej-ZtJiBBv0MtZx2B1MH1ZRXno4s94aL_Qs5u-WCA,32044
 mlrun/db/factory.py,sha256=yP2vVmveUE7LYTCHbS6lQIxP9rW--zdISWuPd_I3d_4,2111
-mlrun/db/httpdb.py,sha256=W31bjXQeN7U5r1eXRxHSRJFIS3jZAVo50ycaFh9mgog,238177
-mlrun/db/nopdb.py,sha256=xWmaw--A1t6zFGX7o7bkIfom5FZpv80vx5V3w-a0P5o,28073
+mlrun/db/httpdb.py,sha256=MiXRygol2qMIMVLDOB89pouR8oICDaR510rVpZtPc4w,239160
+mlrun/db/nopdb.py,sha256=kRWKEEI9LUskI3mp2ofTdAWVLov-99-nSMdaqhi3XT8,28194
 mlrun/feature_store/__init__.py,sha256=SlI845bWt6xX34SXunHHqhmFAR9-5v2ak8N-qpcAPGo,1328
 mlrun/feature_store/api.py,sha256=qKj5Tk6prTab6XWatWhBuPRVp0eJEctoxRMN2wz48vA,32168
 mlrun/feature_store/common.py,sha256=JlQA7XWkg9fLuw7cXFmWpUneQqM3NBhwv7DU_xlenWI,12819
@@ -222,7 +222,7 @@ mlrun/launcher/__init__.py,sha256=JL8qkT1lLr1YvW6iP0hmwDTaSR2RfrMDx0-1gWRhTOE,57
 mlrun/launcher/base.py,sha256=6T17geeplFgYL2Suu4xo5crN_s9JETtr1m0ael8dIYk,17225
 mlrun/launcher/client.py,sha256=cl40ZdF2fU1QbUKdl4Xnucb1u2h-8_dn095qIUyxbuM,6402
 mlrun/launcher/factory.py,sha256=RW7mfzEFi8fR0M-4W1JQg1iq3_muUU6OTqT_3l4Ubrk,2338
-mlrun/launcher/local.py,sha256=4SHY8d8p-nWWMeFywgBb_uzyct8GFyVMW3ysIvHGPVU,11724
+mlrun/launcher/local.py,sha256=3gv-IQYoIChSmRaZ0vLUh0Tu26oLMCx9GbBYh4fWygQ,12161
 mlrun/launcher/remote.py,sha256=zFXE52Cq_7EkC8lfNKT0ceIbye0CfFiundF7O1YU4Xw,7810
 mlrun/model_monitoring/__init__.py,sha256=qDQnncjya9XPTlfvGyfWsZWiXc-glGZrrNja-5QmCZk,782
 mlrun/model_monitoring/api.py,sha256=lAsUp-gzqw8D1cpHVGA2_nPMYn5R4jdxk9UaGOiQ8fE,25945
@@ -276,8 +276,8 @@ mlrun/platforms/__init__.py,sha256=ZuyeHCHHUxYEoZRmaJqzFSfwhaTyUdBZXMeVp75ql1w,3
 mlrun/platforms/iguazio.py,sha256=6VBTq8eQ3mzT96tzjYhAtcMQ2VjF4x8LpIPW5DAcX2Q,13749
 mlrun/projects/__init__.py,sha256=hdCOA6_fp8X4qGGGT7Bj7sPbkM1PayWuaVZL0DkpuZw,1240
 mlrun/projects/operations.py,sha256=Rc__P5ucNAY2G-lHc2LrnZs15PUbNFt8-NqNNT2Bjpk,20623
-mlrun/projects/pipelines.py,sha256=kY5BUHAjNri-9KjWZiCZ9Wo5XwZFqpvqctWy5j8va60,51611
-mlrun/projects/project.py,sha256=FabJ7Eq19PBmlGMv0Rg1XimqJdpciK_cNrkAlzDyvHs,253966
+mlrun/projects/pipelines.py,sha256=nGDzBABEOqoe9sWbax4SfF8CVLgrvK0NLWBadzEthVE,52219
+mlrun/projects/project.py,sha256=B2KCd5bAGMeHOGhEdVr5NKtTOigL6O-A-Bbs2FLvokY,253884
 mlrun/runtimes/__init__.py,sha256=8cqrYKy1a0_87XG7V_p96untQ4t8RocadM4LVEEN1JM,9029
 mlrun/runtimes/base.py,sha256=FVEooeQMpwxIK2iW1R0FNbC5P1sZ_efKtJcsdNSYNmc,38266
 mlrun/runtimes/daskjob.py,sha256=kR5sDQtXtXY_VGn5Y3mapjEEB5P6Lj30pSrPe1DqsAg,20077
@@ -289,7 +289,7 @@ mlrun/runtimes/local.py,sha256=R72VdrXnFdAhLsKJiWPOcfsi4jS-W5E1FnkT2Xllt8M,22150
 mlrun/runtimes/mounts.py,sha256=2dkoktm3TXHe4XHmRhvC0UfvWzq2vy_13MeaW7wgyPo,18735
 mlrun/runtimes/pod.py,sha256=AlA8B8OhyhZyP-C8Gvik_RxoVZsGSXgA7kZado82AQo,52253
 mlrun/runtimes/remotesparkjob.py,sha256=BalAea66GleaKeoYTw6ZL1Qr4wf1yRxfgk1-Fkc9Pqg,7864
-mlrun/runtimes/utils.py,sha256=VFKA7dWuILAcJGia_7Pw_zBBG00wZlat7o2N6u5EItw,16284
+mlrun/runtimes/utils.py,sha256=ofHEOPiMVpQmgxnDhzpHoXBIS97q7QVHiN-UxVSMZkQ,16158
 mlrun/runtimes/databricks_job/__init__.py,sha256=kXGBqhLN0rlAx0kTXhozGzFsIdSqW0uTSKMmsLgq_is,569
 mlrun/runtimes/databricks_job/databricks_cancel_task.py,sha256=ufjcLKA5E6FSDF5CXm5l8uP_mUSFppwr5krLHln1kAU,2243
 mlrun/runtimes/databricks_job/databricks_runtime.py,sha256=ceX0umkNMHvxuXZic4QulWOfJyhPKHVo3T-oPhKTO8Y,12874
@@ -311,10 +311,10 @@ mlrun/serving/__init__.py,sha256=nriJAcVn5aatwU03T7SsE6ngJEGTxr3wIGt4WuvCCzY,139
 mlrun/serving/merger.py,sha256=pfOQoozUyObCTpqXAMk94PmhZefn4bBrKufO3MKnkAc,6193
 mlrun/serving/remote.py,sha256=Igha2FipK3-6rV_PZ1K464kTbiTu8rhc6SMm-HiEJ6o,18817
 mlrun/serving/routers.py,sha256=SmBOlHn7rT2gWTa-W8f16UB0UthgIFc4D1cPOZAA9ss,54003
-mlrun/serving/server.py,sha256=LnnJsjPOh50p5rEPPgvWSP_UQ89JE75q7MbpvdJ1U6c,35221
+mlrun/serving/server.py,sha256=aFhPlIauww1e2OleZdrQxRiqMmEsoHEdTSujnAZeNe4,35202
 mlrun/serving/serving_wrapper.py,sha256=UL9hhWCfMPcTJO_XrkvNaFvck1U1E7oS8trTZyak0cA,835
-mlrun/serving/states.py,sha256=C7A6ziq3XzXzBqLRTSXP2W372PgNe0AptX6qmOwGs-g,120783
-mlrun/serving/system_steps.py,sha256=9AqSQwv6nVljGKZoWJbksnuqsl3VqETcytEwjEVLmA4,16446
+mlrun/serving/states.py,sha256=ZKhlPRlyvbRaGqNueR74IP3q7n8_k-4lYzq0QhPp88Q,124767
+mlrun/serving/system_steps.py,sha256=G-vFdFXGDMZECasT831GxAayitCORNfpxPPRgK5a86w,17960
 mlrun/serving/utils.py,sha256=Zbfqm8TKNcTE8zRBezVBzpvR2WKeKeIRN7otNIaiYEc,4170
 mlrun/serving/v1_serving.py,sha256=c6J_MtpE-Tqu00-6r4eJOCO6rUasHDal9W2eBIcrl50,11853
 mlrun/serving/v2_serving.py,sha256=257LVOvWxV0KjeY0-Kxro6YgKmPu2QzNne2IORlXi5E,25434
@@ -328,7 +328,7 @@ mlrun/utils/async_http.py,sha256=8Olx8TNNeXB07nEGwlqhEgFgnFAD71vBU_bqaA9JW-w,122
 mlrun/utils/azure_vault.py,sha256=IEFizrDGDbAaoWwDr1WoA88S_EZ0T--vjYtY-i0cvYQ,3450
 mlrun/utils/clones.py,sha256=qbAGyEbSvlewn3Tw_DpQZP9z6MGzFhSaZfI1CblX8Fg,7515
 mlrun/utils/condition_evaluator.py,sha256=-nGfRmZzivn01rHTroiGY4rqEv8T1irMyhzxEei-sKc,1897
-mlrun/utils/helpers.py,sha256=HWmSA-5iPt6cduq6DPtYwZJLksnfDj_YSc0kGmRzoxE,80974
+mlrun/utils/helpers.py,sha256=ympa9GWzF_NMNemKSas29s8I0-2seHGhfErFT1b-6tY,82419
 mlrun/utils/http.py,sha256=5ZU2VpokaUM_DT3HBSqTm8xjUqTPjZN5fKkSIvKlTl0,8704
 mlrun/utils/logger.py,sha256=RG0m1rx6gfkJ-2C1r_p41MMpPiaDYqaYM2lYHDlNZEU,14767
 mlrun/utils/regex.py,sha256=FcRwWD8x9X3HLhCCU2F0AVKTFah784Pr7ZAe3a02jw8,5199
@@ -347,11 +347,11 @@ mlrun/utils/notifications/notification/mail.py,sha256=ZyJ3eqd8simxffQmXzqd3bgbAq
 mlrun/utils/notifications/notification/slack.py,sha256=kfhogR5keR7Zjh0VCjJNK3NR5_yXT7Cv-x9GdOUW4Z8,7294
 mlrun/utils/notifications/notification/webhook.py,sha256=zxh8CAlbPnTazsk6r05X5TKwqUZVOH5KBU2fJbzQlG4,5330
 mlrun/utils/version/__init__.py,sha256=YnzE6tlf24uOQ8y7Z7l96QLAI6-QEii7-77g8ynmzy0,613
-mlrun/utils/version/version.json,sha256=smDmVYLs66w3gRDkSXNYS-ogEoZt0cnZOIMpDT7DVK4,90
+mlrun/utils/version/version.json,sha256=yzjgOoSOXuxZFECc2IfKerszx07aKIlhF7wENYuFF_E,90
 mlrun/utils/version/version.py,sha256=M2hVhRrgkN3SxacZHs3ZqaOsqAA7B6a22ne324IQ1HE,1877
-mlrun-1.10.0rc15.dist-info/licenses/LICENSE,sha256=zTiv1CxWNkOk1q8eJS1G_8oD4gWpWLwWxj_Agcsi8Os,11337
-mlrun-1.10.0rc15.dist-info/METADATA,sha256=Vg4ywmTMJAwY8dH__8d4KxE0XJiaExXtqZygOggFAoo,26195
-mlrun-1.10.0rc15.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mlrun-1.10.0rc15.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
-mlrun-1.10.0rc15.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.10.0rc15.dist-info/RECORD,,
+mlrun-1.10.0rc16.dist-info/licenses/LICENSE,sha256=zTiv1CxWNkOk1q8eJS1G_8oD4gWpWLwWxj_Agcsi8Os,11337
+mlrun-1.10.0rc16.dist-info/METADATA,sha256=U1uo_EFf7k8D102Lq0z_EQ1K2AOCjwU5iNQuLVvCVjM,26195
+mlrun-1.10.0rc16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mlrun-1.10.0rc16.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
+mlrun-1.10.0rc16.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.10.0rc16.dist-info/RECORD,,