mlrun 1.7.0rc35__py3-none-any.whl → 1.7.0rc37__py3-none-any.whl

mlrun/model_monitoring/writer.py

@@ -130,7 +130,6 @@ class ModelMonitoringWriter(StepToDict):
         project_name: str,
         result_kind: int,
     ) -> None:
-        logger.info("Sending an event")
         entity = mlrun.common.schemas.alert.EventEntities(
             kind=alert_objects.EventEntityKind.MODEL_ENDPOINT_RESULT,
             project=project_name,
@@ -146,7 +145,9 @@ class ModelMonitoringWriter(StepToDict):
             entity=entity,
             value_dict=event_value,
         )
+        logger.info("Sending a drift event")
         mlrun.get_run_db().generate_event(event_kind, event_data)
+        logger.info("Drift event sent successfully")
 
     @staticmethod
     def _generate_alert_event_kind(
@@ -261,3 +262,5 @@ class ModelMonitoringWriter(StepToDict):
                 endpoint_id=endpoint_id,
                 attributes=json.loads(event[ResultData.RESULT_EXTRA_DATA]),
             )
+
+        logger.info("Model monitoring writer finished handling event")

mlrun/projects/operations.py

@@ -187,6 +187,10 @@ def run_function(
     task.spec.verbose = task.spec.verbose or verbose
 
     if engine == "kfp":
+        if schedule:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Scheduling job is not supported when running a workflow with kfp engine."
+            )
         return function.as_step(
             name=name, runspec=task, workdir=workdir, outputs=outputs, labels=labels
         )

mlrun/projects/project.py

@@ -2967,6 +2967,7 @@ class MlrunProject(ModelObj):
         source: str = None,
         cleanup_ttl: int = None,
         notifications: list[mlrun.model.Notification] = None,
+        send_start_notification: bool = True,
     ) -> _PipelineRunStatus:
         """Run a workflow using kubeflow pipelines
 
@@ -3003,6 +3004,8 @@ class MlrunProject(ModelObj):
                           workflow and all its resources are deleted)
         :param notifications:
                           List of notifications to send for workflow completion
+        :param send_start_notification:
+                          Send a notification when the workflow starts
 
         :returns: ~py:class:`~mlrun.projects.pipelines._PipelineRunStatus` instance
         """
@@ -3080,6 +3083,7 @@ class MlrunProject(ModelObj):
             namespace=namespace,
             source=source,
             notifications=notifications,
+            send_start_notification=send_start_notification,
         )
         # run is None when scheduling
         if run and run.state == mlrun_pipelines.common.models.RunStatuses.failed:

mlrun/runtimes/base.py

@@ -929,3 +929,6 @@ class BaseRuntime(ModelObj):
                         if "default" in p:
                             line += f", default={p['default']}"
                         print("    " + line)
+
+    def skip_image_enrichment(self):
+        return False

mlrun/runtimes/nuclio/api_gateway.py

@@ -657,7 +657,7 @@ class APIGateway(ModelObj):
         host = self.spec.host
         if not self.spec.host.startswith("http"):
             host = f"https://{self.spec.host}"
-        return urljoin(host, self.spec.path)
+        return urljoin(host, self.spec.path).rstrip("/")
 
     @staticmethod
     def _generate_basic_auth(username: str, password: str):

mlrun/runtimes/nuclio/application/application.py

@@ -18,6 +18,7 @@ import nuclio
 
 import mlrun.common.schemas as schemas
 import mlrun.errors
+import mlrun.run
 from mlrun.common.runtimes.constants import NuclioIngressAddTemplatedIngressModes
 from mlrun.runtimes import RemoteRuntime
 from mlrun.runtimes.nuclio import min_nuclio_versions
@@ -174,6 +175,7 @@ class ApplicationStatus(NuclioStatus):
 
 class ApplicationRuntime(RemoteRuntime):
     kind = "application"
+    reverse_proxy_image = None
 
     @min_nuclio_versions("1.13.1")
     def __init__(self, spec=None, metadata=None):
@@ -306,10 +308,11 @@ class ApplicationRuntime(RemoteRuntime):
                 show_on_failure=show_on_failure,
             )
 
-        self._ensure_reverse_proxy_configurations()
+        # This is a class method that accepts a function instance, so we pass self as the function instance
+        self._ensure_reverse_proxy_configurations(self)
         self._configure_application_sidecar()
 
-        # we only allow accessing the application via the API Gateway
+        # We only allow accessing the application via the API Gateway
         name_tag = tag or self.metadata.tag
         self.status.api_gateway_name = (
             f"{self.metadata.name}-{name_tag}" if name_tag else self.metadata.name
@@ -391,8 +394,8 @@ class ApplicationRuntime(RemoteRuntime):
             "main:Handler",
         )
 
-    @classmethod
-    def get_filename_and_handler(cls) -> (str, str):
+    @staticmethod
+    def get_filename_and_handler() -> (str, str):
         reverse_proxy_file_path = pathlib.Path(__file__).parent / "reverse_proxy.go"
         return str(reverse_proxy_file_path), "Handler"
 
@@ -488,6 +491,39 @@ class ApplicationRuntime(RemoteRuntime):
             **http_client_kwargs,
         )
 
+    @classmethod
+    def deploy_reverse_proxy_image(cls):
+        """
+        Build the reverse proxy image and save it.
+        The reverse proxy image is used to route requests to the application sidecar.
+        This is useful when you want to decrease build time by building the application image only once.
+
+        :param use_cache:   Use the cache when building the image
+        """
+        # create a function that includes only the reverse proxy, without the application
+
+        reverse_proxy_func = mlrun.run.new_function(
+            name="reverse-proxy-temp", kind="remote"
+        )
+        # default max replicas is 4, we only need one replica for the reverse proxy
+        reverse_proxy_func.spec.max_replicas = 1
+
+        # the reverse proxy image should not be based on another image
+        reverse_proxy_func.set_config("spec.build.baseImage", None)
+        reverse_proxy_func.spec.image = ""
+        reverse_proxy_func.spec.build.base_image = ""
+
+        cls._ensure_reverse_proxy_configurations(reverse_proxy_func)
+        reverse_proxy_func.deploy()
+
+        # save the created container image
+        cls.reverse_proxy_image = reverse_proxy_func.status.container_image
+
+        # delete the function to avoid cluttering the project
+        mlrun.get_run_db().delete_function(
+            reverse_proxy_func.metadata.name, reverse_proxy_func.metadata.project
+        )
+
     def _run(self, runobj: "mlrun.RunObject", execution):
         raise mlrun.runtimes.RunError(
             "Application runtime .run() is not yet supported. Use .invoke() instead."
@@ -527,21 +563,22 @@ class ApplicationRuntime(RemoteRuntime):
             with_mlrun=with_mlrun,
         )
 
-    def _ensure_reverse_proxy_configurations(self):
-        if self.spec.build.functionSourceCode or self.status.container_image:
+    @staticmethod
+    def _ensure_reverse_proxy_configurations(function: RemoteRuntime):
+        if function.spec.build.functionSourceCode or function.status.container_image:
             return
 
         filename, handler = ApplicationRuntime.get_filename_and_handler()
         name, spec, code = nuclio.build_file(
             filename,
-            name=self.metadata.name,
+            name=function.metadata.name,
             handler=handler,
         )
-        self.spec.function_handler = mlrun.utils.get_in(spec, "spec.handler")
-        self.spec.build.functionSourceCode = mlrun.utils.get_in(
+        function.spec.function_handler = mlrun.utils.get_in(spec, "spec.handler")
+        function.spec.build.functionSourceCode = mlrun.utils.get_in(
             spec, "spec.build.functionSourceCode"
         )
-        self.spec.nuclio_runtime = mlrun.utils.get_in(spec, "spec.runtime")
+        function.spec.nuclio_runtime = mlrun.utils.get_in(spec, "spec.runtime")
 
     def _configure_application_sidecar(self):
         # Save the application image in the status to allow overriding it with the reverse proxy entry point
@@ -552,8 +589,12 @@ class ApplicationRuntime(RemoteRuntime):
             self.status.application_image = self.spec.image
             self.spec.image = ""
 
-        if self.status.container_image:
-            self.from_image(self.status.container_image)
+        # reuse the reverse proxy image if it was built before
+        if (
+            reverse_proxy_image := self.status.container_image
+            or self.reverse_proxy_image
+        ):
+            self.from_image(reverse_proxy_image)
 
         self.status.sidecar_name = f"{self.metadata.name}-sidecar"
         self.with_sidecar(

mlrun/runtimes/nuclio/function.py

@@ -689,7 +689,7 @@ class RemoteRuntime(KubeResource):
             "State thresholds do not apply for nuclio as it has its own function pods healthiness monitoring"
         )
 
-    @min_nuclio_versions("1.12.8")
+    @min_nuclio_versions("1.13.1")
     def disable_default_http_trigger(
         self,
     ):
@@ -707,6 +707,10 @@ class RemoteRuntime(KubeResource):
         """
         self.spec.disable_default_http_trigger = False
 
+    def skip_image_enrichment(self):
+        # make sure the API does not enrich the base image if the function is not a python function
+        return self.spec.nuclio_runtime and "python" not in self.spec.nuclio_runtime
+
     def _get_state(
         self,
         dashboard="",

mlrun/runtimes/sparkjob/spark3job.py

@@ -505,13 +505,10 @@ class Spark3Runtime(KubejobRuntime):
             raise NotImplementedError(
                 "Setting node name is not supported for spark runtime"
             )
-        # TODO add affinity support
-        # https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go#L491
-        if affinity:
-            raise NotImplementedError(
-                "Setting affinity is not supported for spark runtime"
-            )
-        super().with_node_selection(node_name, node_selector, affinity, tolerations)
+        self.with_driver_node_selection(node_name, node_selector, affinity, tolerations)
+        self.with_executor_node_selection(
+            node_name, node_selector, affinity, tolerations
+        )
 
     def with_driver_node_selection(
         self,

mlrun/runtimes/utils.py

@@ -445,3 +445,21 @@ def enrich_run_labels(
         if label.value not in labels and enrichment:
             labels[label.value] = enrichment
     return labels
+
+
+def resolve_node_selectors(
+    project_node_selector: dict, instance_node_selector: dict
+) -> dict:
+    config_node_selector = mlrun.mlconf.get_default_function_node_selector()
+    if project_node_selector or config_node_selector:
+        mlrun.utils.logger.debug(
+            "Enriching node selector from project and mlrun config",
+            project_node_selector=project_node_selector,
+            config_node_selector=config_node_selector,
+        )
+        return mlrun.utils.helpers.merge_dicts_with_precedence(
+            config_node_selector,
+            project_node_selector,
+            instance_node_selector,
+        )
+    return instance_node_selector

mlrun/serving/routers.py

@@ -32,7 +32,6 @@ from mlrun.errors import err_to_str
 from mlrun.utils import logger, now_date
 
 from ..common.helpers import parse_versioned_object_uri
-from ..config import config
 from .server import GraphServer
 from .utils import RouterToDict, _extract_input_data, _update_result_body
 from .v2_serving import _ModelLogPusher
@@ -1057,9 +1056,7 @@ def _init_endpoint_record(
                 function_uri=graph_server.function_uri,
                 model=versioned_model_name,
                 model_class=voting_ensemble.__class__.__name__,
-                stream_path=config.model_endpoint_monitoring.store_prefixes.default.format(
-                    project=project, kind="stream"
-                ),
+                stream_path=voting_ensemble.context.stream.stream_uri,
                 active=True,
                 monitoring_mode=mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled,
             ),

mlrun/serving/server.py

@@ -38,10 +38,7 @@ from ..errors import MLRunInvalidArgumentError
 from ..model import ModelObj
 from ..utils import get_caller_globals
 from .states import RootFlowStep, RouterStep, get_function, graph_root_setter
-from .utils import (
-    event_id_key,
-    event_path_key,
-)
+from .utils import event_id_key, event_path_key
 
 
 class _StreamContext:
@@ -71,15 +68,15 @@ class _StreamContext:
                 function_uri, config.default_project
             )
 
-            stream_uri = mlrun.model_monitoring.get_stream_path(project=project)
+            self.stream_uri = mlrun.model_monitoring.get_stream_path(project=project)
 
             if log_stream:
                 # Update the stream path to the log stream value
-                stream_uri = log_stream.format(project=project)
+                self.stream_uri = log_stream.format(project=project)
 
             stream_args = parameters.get("stream_args", {})
 
-            self.output_stream = get_stream_pusher(stream_uri, **stream_args)
+            self.output_stream = get_stream_pusher(self.stream_uri, **stream_args)
 
 
 class GraphServer(ModelObj):

mlrun/serving/states.py

@@ -872,7 +872,8 @@ class QueueStep(BaseStep):
             return event
 
         if self._stream:
-            if self.options.get("full_event", True):
+            full_event = self.options.get("full_event")
+            if full_event or full_event is None and self.next:
                 data = storey.utils.wrap_event_for_serialization(event, data)
             self._stream.push(data)
             event.terminated = True
@@ -1390,7 +1391,7 @@ class FlowStep(BaseStep):
         return step
 
     def supports_termination(self):
-        return self.engine == "async"
+        return self.engine != "sync"
 
 
 class RootFlowStep(FlowStep):
@@ -1630,7 +1631,11 @@ def _init_async_objects(context, steps):
                 if step.path and not skip_stream:
                     stream_path = step.path
                     endpoint = None
-                    options = {}
+                    # in case of a queue, we default to a full_event=True
+                    full_event = step.options.get("full_event")
+                    options = {
+                        "full_event": full_event or full_event is None and step.next
+                    }
                     options.update(step.options)
 
                     kafka_brokers = get_kafka_brokers_from_dict(options, pop=True)

mlrun/serving/v2_serving.py

@@ -15,12 +15,11 @@
 import threading
 import time
 import traceback
-from typing import Union
+from typing import Optional, Union
 
+import mlrun.artifacts
 import mlrun.common.model_monitoring
 import mlrun.common.schemas.model_monitoring
-from mlrun.artifacts import ModelArtifact  # noqa: F401
-from mlrun.config import config
 from mlrun.errors import err_to_str
 from mlrun.utils import logger, now_date
 
@@ -102,7 +101,7 @@ class V2ModelServer(StepToDict):
         self.error = ""
         self.protocol = protocol or "v2"
         self.model_path = model_path
-        self.model_spec: mlrun.artifacts.ModelArtifact = None
+        self.model_spec: Optional[mlrun.artifacts.ModelArtifact] = None
         self._input_path = input_path
         self._result_path = result_path
         self._kwargs = kwargs  # for to_dict()
@@ -335,6 +334,7 @@ class V2ModelServer(StepToDict):
             else:
                 track_request = {"id": event_id, "inputs": inputs or []}
                 track_response = {"outputs": outputs or []}
+                # TODO : check dict/list
                 self._model_logger.push(start, track_request, track_response, op)
         event.body = _update_result_body(self._result_path, original_body, response)
         return event
@@ -376,8 +376,10 @@ class V2ModelServer(StepToDict):
         """postprocess, before returning response"""
         return request
 
-    def predict(self, request: dict) -> dict:
-        """model prediction operation"""
+    def predict(self, request: dict) -> list:
+        """model prediction operation
+        :return: list with the model prediction results (can be multi-port) or list of lists for multiple predictions
+        """
         raise NotImplementedError()
 
     def explain(self, request: dict) -> dict:
@@ -567,9 +569,7 @@ def _init_endpoint_record(
                 model=versioned_model_name,
                 model_class=model.__class__.__name__,
                 model_uri=model.model_path,
-                stream_path=config.model_endpoint_monitoring.store_prefixes.default.format(
-                    project=project, kind="stream"
-                ),
+                stream_path=model.context.stream.stream_uri,
                 active=True,
                 monitoring_mode=mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled,
             ),

mlrun/utils/db.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import abc
 import pickle
 from datetime import datetime
 
@@ -37,6 +38,13 @@ class BaseModel:
 
         return dict(map(get_key_value, columns))
 
+    @abc.abstractmethod
+    def get_identifier_string(self):
+        """
+        This method must be implemented by any subclass.
+        """
+        pass
+
 
 class HasStruct(BaseModel):
     @property
@@ -54,3 +62,10 @@ class HasStruct(BaseModel):
         exclude = exclude or []
         exclude.append("body")
         return super().to_dict(exclude, strip=strip)
+
+    @abc.abstractmethod
+    def get_identifier_string(self):
+        """
+        This method must be implemented by any subclass.
+        """
+        pass

mlrun/utils/http.py

@@ -95,7 +95,7 @@ class HTTPSessionWithRetry(requests.Session):
                     total=self.max_retries,
                     backoff_factor=self.retry_backoff_factor,
                     status_forcelist=config.http_retry_defaults.status_codes,
-                    method_whitelist=self._retry_methods,
+                    allowed_methods=self._retry_methods,
                     # we want to retry but not to raise since we do want that last response (to parse details on the
                     # error from response body) we'll handle raising ourselves
                     raise_on_status=False,

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "b2082382ddf7988e610a7ab6f9ea1a0ff8da863c",
-  "version": "1.7.0-rc35"
+  "git_commit": "9edc20898a23dc8a02ea4c3f6db2e20054146ad8",
+  "version": "1.7.0-rc37"
 }

{mlrun-1.7.0rc35.dist-info → mlrun-1.7.0rc37.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.7.0rc35
+Version: 1.7.0rc37
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -28,7 +28,7 @@ Requires-Dist: aiohttp-retry ~=2.8
 Requires-Dist: click ~=8.1
 Requires-Dist: nest-asyncio ~=1.0
 Requires-Dist: ipython ~=8.10
-Requires-Dist: nuclio-jupyter ~=0.10.3
+Requires-Dist: nuclio-jupyter ~=0.10.4
 Requires-Dist: numpy <1.27.0,>=1.16.5
 Requires-Dist: pandas <2.2,>=1.2
 Requires-Dist: pyarrow <15,>=10.0
@@ -50,8 +50,8 @@ Requires-Dist: setuptools ~=71.0
 Requires-Dist: deprecated ~=1.2
 Requires-Dist: jinja2 >=3.1.3,~=3.1
 Requires-Dist: orjson <4,>=3.9.15
-Requires-Dist: mlrun-pipelines-kfp-common ~=0.1.3
-Requires-Dist: mlrun-pipelines-kfp-v1-8 ~=0.1.3
+Requires-Dist: mlrun-pipelines-kfp-common ~=0.1.6
+Requires-Dist: mlrun-pipelines-kfp-v1-8 ~=0.1.6
 Provides-Extra: alibaba-oss
 Requires-Dist: ossfs ==2023.12.0 ; extra == 'alibaba-oss'
 Requires-Dist: oss2 ==2.18.1 ; extra == 'alibaba-oss'
@@ -97,7 +97,7 @@ Requires-Dist: sqlalchemy ~=1.4 ; extra == 'api'
 Requires-Dist: pymysql ~=1.0 ; extra == 'api'
 Requires-Dist: alembic ~=1.9 ; extra == 'api'
 Requires-Dist: timelength ~=1.1 ; extra == 'api'
-Requires-Dist: memray ~=1.12 ; extra == 'api'
+Requires-Dist: memray ~=1.12 ; (sys_platform != "win32") and extra == 'api'
 Provides-Extra: azure-blob-storage
 Requires-Dist: msrest ~=0.6.21 ; extra == 'azure-blob-storage'
 Requires-Dist: azure-core ~=1.24 ; extra == 'azure-blob-storage'
@@ -156,7 +156,6 @@ Requires-Dist: graphviz ~=0.20.0 ; extra == 'complete-api'
 Requires-Dist: humanfriendly ~=10.0 ; extra == 'complete-api'
 Requires-Dist: igz-mgmt ~=0.2.0 ; extra == 'complete-api'
 Requires-Dist: kafka-python ~=2.0 ; extra == 'complete-api'
-Requires-Dist: memray ~=1.12 ; extra == 'complete-api'
 Requires-Dist: mlflow ~=2.8 ; extra == 'complete-api'
 Requires-Dist: msrest ~=0.6.21 ; extra == 'complete-api'
 Requires-Dist: objgraph ~=3.6 ; extra == 'complete-api'
@@ -172,6 +171,7 @@ Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete-api'
 Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'complete-api'
 Requires-Dist: timelength ~=1.1 ; extra == 'complete-api'
 Requires-Dist: uvicorn ~=0.27.1 ; extra == 'complete-api'
+Requires-Dist: memray ~=1.12 ; (sys_platform != "win32") and extra == 'complete-api'
 Provides-Extra: dask
 Requires-Dist: dask ~=2023.9.0 ; extra == 'dask'
 Requires-Dist: distributed ~=2023.9.0 ; extra == 'dask'