mlrun 1.10.0rc1__py3-none-any.whl → 1.10.0rc3__py3-none-any.whl

mlrun/runtimes/nuclio/serving.py

@@ -11,12 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import copy
 import json
 import os
 import warnings
 from copy import deepcopy
-from typing import TYPE_CHECKING, Optional, Union
+from typing import Optional, Union
 
 import nuclio
 from nuclio import KafkaTrigger
@@ -27,7 +27,11 @@ from mlrun.datastore import get_kafka_brokers_from_dict, parse_kafka_url
 from mlrun.model import ObjectList
 from mlrun.runtimes.function_reference import FunctionReference
 from mlrun.secrets import SecretsStore
-from mlrun.serving.server import GraphServer, create_graph_server
+from mlrun.serving.server import (
+    GraphServer,
+    add_system_steps_to_graph,
+    create_graph_server,
+)
 from mlrun.serving.states import (
     RootFlowStep,
     RouterStep,
@@ -43,10 +47,6 @@ from .function import NuclioSpec, RemoteRuntime, min_nuclio_versions
 
 serving_subkind = "serving_v2"
 
-if TYPE_CHECKING:
-    # remove this block in 1.9.0
-    from mlrun.model_monitoring import TrackingPolicy
-
 
 def new_v2_model_server(
     name,
@@ -95,7 +95,6 @@ class ServingSpec(NuclioSpec):
         "default_class",
         "secret_sources",
         "track_models",
-        "tracking_policy",
     ]
 
     def __init__(
@@ -132,7 +131,6 @@ class ServingSpec(NuclioSpec):
         graph_initializer=None,
         error_stream=None,
         track_models=None,
-        tracking_policy=None,
         secret_sources=None,
         default_content_type=None,
         node_name=None,
@@ -207,7 +205,6 @@ class ServingSpec(NuclioSpec):
         self.graph_initializer = graph_initializer
         self.error_stream = error_stream
         self.track_models = track_models
-        self.tracking_policy = tracking_policy
         self.secret_sources = secret_sources or []
         self.default_content_type = default_content_type
         self.model_endpoint_creation_task_name = model_endpoint_creation_task_name
@@ -314,7 +311,6 @@ class ServingRuntime(RemoteRuntime):
         batch: Optional[int] = None,
         sampling_percentage: float = 100,
         stream_args: Optional[dict] = None,
-        tracking_policy: Optional[Union["TrackingPolicy", dict]] = None,
         enable_tracking: bool = True,
     ) -> None:
         """Apply on your serving function to monitor a deployed model, including real-time dashboards to detect drift
@@ -361,20 +357,12 @@ class ServingRuntime(RemoteRuntime):
         if batch:
             warnings.warn(
                 "The `batch` size parameter was deprecated in version 1.8.0 and is no longer used. "
-                "It will be removed in 1.10.",
-                # TODO: Remove this in 1.10
+                "It will be removed in 1.11.",
+                # TODO: Remove this in 1.11
                 FutureWarning,
             )
         if stream_args:
             self.spec.parameters["stream_args"] = stream_args
-        if tracking_policy is not None:
-            warnings.warn(
-                "The `tracking_policy` argument is deprecated from version 1.7.0 "
-                "and has no effect. It will be removed in 1.9.0.\n"
-                "To set the desired model monitoring time window and schedule, use "
-                "the `base_period` argument in `project.enable_model_monitoring()`.",
-                FutureWarning,
-            )
 
     def add_model(
         self,
@@ -719,7 +707,6 @@ class ServingRuntime(RemoteRuntime):
             "graph_initializer": self.spec.graph_initializer,
             "error_stream": self.spec.error_stream,
             "track_models": self.spec.track_models,
-            "tracking_policy": None,
             "default_content_type": self.spec.default_content_type,
             "model_endpoint_creation_task_name": self.spec.model_endpoint_creation_task_name,
         }
@@ -761,10 +748,13 @@ class ServingRuntime(RemoteRuntime):
             set_paths(workdir)
             os.chdir(workdir)
 
+        system_graph = None
+        if isinstance(self.spec.graph, RootFlowStep):
+            system_graph = add_system_steps_to_graph(copy.deepcopy(self.spec.graph))
         server = create_graph_server(
             parameters=self.spec.parameters,
             load_mode=self.spec.load_mode,
-            graph=self.spec.graph,
+            graph=system_graph or self.spec.graph,
             verbose=self.verbose,
             current_function=current_function,
             graph_initializer=self.spec.graph_initializer,

mlrun/runtimes/utils.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import enum
 import getpass
 import hashlib
 import json
@@ -28,7 +29,6 @@ import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
 import mlrun.utils.regex
 from mlrun.artifacts import TableArtifact
-from mlrun.common.runtimes.constants import RunLabels
 from mlrun.config import config
 from mlrun.errors import err_to_str
 from mlrun.frameworks.parallel_coordinates import gen_pcp_plot
@@ -433,18 +433,35 @@ def enrich_function_from_dict(function, function_dict):
 
 def enrich_run_labels(
     labels: dict,
-    labels_to_enrich: Optional[list[RunLabels]] = None,
+    labels_to_enrich: Optional[list[mlrun_constants.MLRunInternalLabels]] = None,
 ):
+    """
+    Enrich the run labels with the internal labels and the labels enrichment extension
+    :param labels: The run labels dict
+    :param labels_to_enrich: The label keys to enrich from MLRunInternalLabels.default_run_labels_to_enrich
+    :return: The enriched labels dict
+    """
+    # Merge the labels with the labels enrichment extension
     labels_enrichment = {
-        RunLabels.owner: os.environ.get("V3IO_USERNAME") or getpass.getuser(),
-        # TODO: remove this in 1.9.0
-        RunLabels.v3io_user: os.environ.get("V3IO_USERNAME"),
+        mlrun_constants.MLRunInternalLabels.owner: os.environ.get("V3IO_USERNAME")
+        or getpass.getuser(),
+        # TODO: remove this in 1.10.0
+        mlrun_constants.MLRunInternalLabels.v3io_user: os.environ.get("V3IO_USERNAME"),
     }
-    labels_to_enrich = labels_to_enrich or RunLabels.all()
+
+    # Resolve which label keys to enrich
+    if labels_to_enrich is None:
+        labels_to_enrich = (
+            mlrun_constants.MLRunInternalLabels.default_run_labels_to_enrich()
+        )
+
+    # Enrich labels
     for label in labels_to_enrich:
+        if isinstance(label, enum.Enum):
+            label = label.value
         enrichment = labels_enrichment.get(label)
-        if label.value not in labels and enrichment:
-            labels[label.value] = enrichment
+        if label not in labels and enrichment:
+            labels[label] = enrichment
     return labels
 
 

mlrun/serving/__init__.py

@@ -30,7 +30,11 @@ __all__ = [
 ]
 
 from .routers import ModelRouter, VotingEnsemble  # noqa
-from .server import GraphContext, GraphServer, create_graph_server  # noqa
+from .server import (
+    GraphContext,
+    GraphServer,
+    create_graph_server,
+)  # noqa
 from .states import (
     ErrorStep,
     QueueStep,

mlrun/serving/server.py

@@ -15,6 +15,7 @@
 __all__ = ["GraphServer", "create_graph_server", "GraphContext", "MockEvent"]
 
 import asyncio
+import copy
 import json
 import os
 import socket
@@ -71,7 +72,7 @@ class _StreamContext:
         if (enabled or log_stream) and function_uri:
             self.enabled = True
             project, _, _, _ = parse_versioned_object_uri(
-                function_uri, config.default_project
+                function_uri, config.active_project
             )
 
             stream_args = parameters.get("stream_args", {})
@@ -108,7 +109,6 @@ class GraphServer(ModelObj):
         graph_initializer=None,
         error_stream=None,
         track_models=None,
-        tracking_policy=None,
         secret_sources=None,
         default_content_type=None,
         function_name=None,
@@ -129,7 +129,6 @@ class GraphServer(ModelObj):
         self.graph_initializer = graph_initializer
         self.error_stream = error_stream
         self.track_models = track_models
-        self.tracking_policy = tracking_policy
         self._error_stream_object = None
         self.secret_sources = secret_sources
         self._secrets = SecretsStore.from_list(secret_sources)
@@ -330,12 +329,49 @@ class GraphServer(ModelObj):
         return self.graph.wait_for_completion()
 
 
+def add_system_steps_to_graph(graph: RootFlowStep):
+    model_runner_raisers = {}
+    steps = list(graph.steps.values())
+    for step in steps:
+        if (
+            isinstance(step, mlrun.serving.states.ModelRunnerStep)
+            and step.raise_exception
+        ):
+            error_step = graph.add_step(
+                class_name="mlrun.serving.states.ModelRunnerErrorRaiser",
+                name=f"{step.name}_error_raise",
+                after=step.name,
+                full_event=True,
+                raise_exception=step.raise_exception,
+                models_names=list(step.class_args["models"].keys()),
+            )
+            if step.responder:
+                step.responder = False
+                error_step.respond()
+            model_runner_raisers[step.name] = error_step.name
+            error_step.on_error = step.on_error
+        if isinstance(step.after, list):
+            for i in range(len(step.after)):
+                if step.after[i] in model_runner_raisers:
+                    step.after[i] = model_runner_raisers[step.after[i]]
+        else:
+            if step.after in model_runner_raisers:
+                step.after = model_runner_raisers[step.after]
+    return graph
+
+
 def v2_serving_init(context, namespace=None):
     """hook for nuclio init_context()"""
 
     context.logger.info("Initializing server from spec")
     spec = mlrun.utils.get_serving_spec()
     server = GraphServer.from_dict(spec)
+    if isinstance(server.graph, RootFlowStep):
+        server.graph = add_system_steps_to_graph(copy.deepcopy(server.graph))
+        context.logger.info_with(
+            "Server graph after adding system steps",
+            graph=str(server.graph.steps),
+        )
 
     if config.log_level.lower() == "debug":
         server.verbose = True

mlrun/serving/states.py

@@ -30,6 +30,7 @@ from typing import Any, Optional, Union, cast
 import storey.utils
 
 import mlrun
+import mlrun.artifacts
 import mlrun.common.schemas as schemas
 from mlrun.datastore.datastore_profile import (
     DatastoreProfileKafkaSource,
@@ -46,7 +47,7 @@ from ..datastore.utils import (
     get_kafka_brokers_from_dict,
     parse_kafka_url,
 )
-from ..errors import MLRunInvalidArgumentError, err_to_str
+from ..errors import MLRunInvalidArgumentError, ModelRunnerError, err_to_str
 from ..model import ModelObj, ObjectDict
 from ..platforms.iguazio import parse_path
 from ..utils import get_class, get_function, is_explicit_ack_supported
@@ -402,6 +403,9 @@ class BaseStep(ModelObj):
             class_args=class_args,
             model_endpoint_creation_strategy=model_endpoint_creation_strategy,
         )
+
+        self.verify_model_runner_step(step)
+
         step = parent._steps.update(name, step)
         step.set_parent(parent)
         if not hasattr(self, "steps"):
@@ -446,6 +450,36 @@ class BaseStep(ModelObj):
     def supports_termination(self):
         return False
 
+    def verify_model_runner_step(self, step: "ModelRunnerStep"):
+        """
+        Verify ModelRunnerStep, can be part of Flow graph and models can not repeat in graph.
+        :param step: ModelRunnerStep to verify
+        """
+        if not isinstance(step, ModelRunnerStep):
+            return
+
+        root = self
+        while root.parent is not None:
+            root = root.parent
+
+        if not isinstance(root, RootFlowStep):
+            raise GraphError(
+                "ModelRunnerStep can be added to 'Flow' topology graph only"
+            )
+        step_model_endpoints_names = list(
+            step.class_args[schemas.ModelRunnerStepData.MODELS].keys()
+        )
+        # Get all model_endpoints names that are in both lists
+        common_endpoints_names = list(
+            set(root.model_endpoints_names) & set(step_model_endpoints_names)
+        )
+        if common_endpoints_names:
+            raise GraphError(
+                f"The graph already contains the model endpoints named - {common_endpoints_names}."
+            )
+        else:
+            root.extend_model_endpoints_names(step_model_endpoints_names)
+
 
 class TaskStep(BaseStep):
     """task execution step, runs a class or handler"""
@@ -988,14 +1022,18 @@ class ModelRunnerStep(TaskStep, StepToDict):
 
     :param model_selector: ModelSelector instance whose select() method will be used to select models to run on each
       event. Optional. If not passed, all models will be run.
+    :param raise_exception:  If True, an error will be raised when model selection fails or if one of the models raised
+      an error. If False, the error will appear in the output event.
     """
 
     kind = "model_runner"
+    _dict_fields = TaskStep._dict_fields + ["raise_exception"]
 
     def __init__(
         self,
         *args,
         model_selector: Optional[Union[str, ModelSelector]] = None,
+        raise_exception: bool = True,
         **kwargs,
     ):
         super().__init__(
@@ -1004,27 +1042,96 @@ class ModelRunnerStep(TaskStep, StepToDict):
             class_args=dict(model_selector=model_selector),
             **kwargs,
         )
+        self.raise_exception = raise_exception
 
-    def add_model(self, model: Union[str, Model], **model_parameters) -> None:
+    def add_model(
+        self,
+        endpoint_name: str,
+        model_class: str,
+        model_artifact: Optional[Union[str, mlrun.artifacts.ModelArtifact]] = None,
+        labels: Optional[Union[list[str], dict[str, str]]] = None,
+        creation_strategy: Optional[
+            schemas.ModelEndpointCreationStrategy
+        ] = schemas.ModelEndpointCreationStrategy.INPLACE,
+        inputs: Optional[list[str]] = None,
+        outputs: Optional[list[str]] = None,
+        input_path: Optional[str] = None,
+        override: bool = False,
+        **model_parameters,
+    ) -> None:
         """
         Add a Model to this ModelRunner.
 
-        :param model: Model class name or object
-        :param model_parameters: Parameters for model instantiation
+        :param endpoint_name:       str, will identify the model in the ModelRunnerStep, and assign model endpoint name
+        :param model_class:         Model class name
+        :param model_artifact:      model artifact or mlrun model artifact uri
+        :param labels:              model endpoint labels, should be list of str or mapping of str:str
+        :param creation_strategy:   Strategy for creating or updating the model endpoint:
+            * **overwrite**:
+            1. If model endpoints with the same name exist, delete the `latest` one.
+            2. Create a new model endpoint entry and set it as `latest`.
+            * **inplace** (default):
+            1. If model endpoints with the same name exist, update the `latest` entry.
+            2. Otherwise, create a new entry.
+            * **archive**:
+            1. If model endpoints with the same name exist, preserve them.
+            2. Create a new model endpoint with the same name and set it to `latest`.
+        :param inputs:              list of the model inputs (e.g. features) ,if provided will override the inputs that
+                                    been configured in the model artifact, please note that those inputs need to be
+                                    equal in length and order to the inputs that model_class predict method expects
+        :param outputs:             list of the model outputs (e.g. labels) ,if provided will override the outputs that
+                                    been configured in the model artifact, please note that those outputs need to be
+                                    equal to the model_class predict method outputs (length, and order)
+        :param input_path:          input path inside the user event, expect scopes to be defined by dot notation
+                                    (e.g "inputs.my_model_inputs"). expects list or dictionary type object in path.
+        :param override:            bool allow override existing model on the current ModelRunnerStep.
+        :param model_parameters:    Parameters for model instantiation
         """
-        models = self.class_args.get("models", [])
-        models.append((model, model_parameters))
-        self.class_args["models"] = models
+        # TODO allow model_class as Model object as part of ML-9924
+        model_parameters = model_parameters or {}
+        if model_parameters.get("name", endpoint_name) != endpoint_name:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Inconsistent name for model added to ModelRunnerStep."
+            )
+
+        models = self.class_args.get(schemas.ModelRunnerStepData.MODELS, {})
+        if endpoint_name in models and not override:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Model with name {endpoint_name} already exists in this ModelRunnerStep."
+            )
+
+        model_parameters["name"] = endpoint_name
+        monitoring_data = self.class_args.get(
+            schemas.ModelRunnerStepData.MONITORING_DATA, {}
+        )
+        models[endpoint_name] = (model_class, model_parameters)
+        monitoring_data[endpoint_name] = {
+            schemas.MonitoringData.INPUTS: inputs,
+            schemas.MonitoringData.OUTPUTS: outputs,
+            schemas.MonitoringData.INPUT_PATH: input_path,
+            schemas.MonitoringData.CREATION_STRATEGY: creation_strategy,
+            schemas.MonitoringData.LABELS: labels,
+            schemas.MonitoringData.MODEL_PATH: model_artifact.uri
+            if isinstance(model_artifact, mlrun.artifacts.Artifact)
+            else model_artifact,
+        }
+        self.class_args[schemas.ModelRunnerStepData.MODELS] = models
+        self.class_args[schemas.ModelRunnerStepData.MONITORING_DATA] = monitoring_data
 
     def init_object(self, context, namespace, mode="sync", reset=False, **extra_kwargs):
         model_selector = self.class_args.get("model_selector")
-        models = self.class_args.get("models")
+        models = self.class_args.get(schemas.ModelRunnerStepData.MODELS, {})
         if isinstance(model_selector, str):
             model_selector = get_class(model_selector, namespace)()
         model_objects = []
-        for model, model_params in models:
+        for model, model_params in models.values():
             if not isinstance(model, Model):
+                # prevent model predict from raising error
+                model_params["raise_exception"] = False
                 model = get_class(model, namespace)(**model_params)
+            else:
+                # prevent model predict from raising error
+                model._raise_exception = False
             model_objects.append(model)
         self._async_object = ModelRunner(
             model_selector=model_selector,
@@ -1032,6 +1139,29 @@ class ModelRunnerStep(TaskStep, StepToDict):
         )
 
 
+class ModelRunnerErrorRaiser(storey.MapClass):
+    def __init__(self, raise_exception: bool, models_names: list[str], **kwargs):
+        super().__init__(**kwargs)
+        self._raise_exception = raise_exception
+        self._models_names = models_names
+
+    def do(self, event):
+        if self._raise_exception:
+            errors = {}
+            should_raise = False
+            if len(self._models_names) == 1:
+                should_raise = event.body.get("error") is not None
+                errors[self._models_names[0]] = event.body.get("error")
+            else:
+                for model in event.body:
+                    errors[model] = event.body.get(model).get("error")
+                    if errors[model] is not None:
+                        should_raise = True
+            if should_raise:
+                raise ModelRunnerError(models_errors=errors)
+        return event
+
+
 class QueueStep(BaseStep, StepToDict):
     """queue step, implement an async queue or represent a stream"""
 
@@ -1256,6 +1386,8 @@ class FlowStep(BaseStep):
             class_args=class_args,
         )
 
+        self.verify_model_runner_step(step)
+
         after_list = after if isinstance(after, list) else [after]
         for after in after_list:
             self.insert_step(name, step, after, before)
@@ -1676,7 +1808,41 @@ class RootFlowStep(FlowStep):
     """root flow step"""
 
     kind = "root"
-    _dict_fields = ["steps", "engine", "final_step", "on_error"]
+    _dict_fields = [
+        "steps",
+        "engine",
+        "final_step",
+        "on_error",
+        "model_endpoints_names",
+    ]
+
+    def __init__(
+        self,
+        name=None,
+        steps=None,
+        after: Optional[list] = None,
+        engine=None,
+        final_step=None,
+    ):
+        super().__init__(
+            name,
+            steps,
+            after,
+            engine,
+            final_step,
+        )
+        self._models = []
+
+    @property
+    def model_endpoints_names(self) -> list[str]:
+        return self._models
+
+    @model_endpoints_names.setter
+    def model_endpoints_names(self, models: list[str]):
+        self._models = models
+
+    def extend_model_endpoints_names(self, model_endpoints_names: list):
+        self._models.extend(model_endpoints_names)
 
 
 classes_map = {

mlrun/utils/helpers.py

@@ -876,13 +876,18 @@ def enrich_image_url(
     client_version: Optional[str] = None,
     client_python_version: Optional[str] = None,
 ) -> str:
+    image_url = image_url.strip()
+
+    # Add python version tag if needed
+    if image_url == "python" and client_python_version:
+        image_url = f"python:{client_python_version}"
+
     client_version = _convert_python_package_version_to_image_tag(client_version)
     server_version = _convert_python_package_version_to_image_tag(
         mlrun.utils.version.Version().get()["version"]
     )
-    image_url = image_url.strip()
     mlrun_version = config.images_tag or client_version or server_version
-    tag = mlrun_version
+    tag = mlrun_version or ""
 
     # TODO: Remove condition when mlrun/mlrun-kfp image is also supported
     if "mlrun-kfp" not in image_url:
@@ -2226,8 +2231,9 @@ class Workflow:
             namespace=mlrun.mlconf.namespace,
         )
 
-        # arbitrary timeout of 5 seconds, the workflow should be done by now
-        kfp_run = kfp_client.wait_for_run_completion(workflow_id, 5)
+        # arbitrary timeout of 60 seconds, the workflow should be done by now, however sometimes kfp takes a few
+        # seconds to update the workflow status
+        kfp_run = kfp_client.wait_for_run_completion(workflow_id, 60)
         if not kfp_run:
             return None
 

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "4d532f1a4e8dd427fd5f870863084fa564680bd6",
-  "version": "1.10.0-rc1"
+  "git_commit": "210c516a3ed5c2f2c7223f31fdfd9e99b73d56b6",
+  "version": "1.10.0-rc3"
 }