mlrun 1.7.0rc32__py3-none-any.whl → 1.7.0rc34__py3-none-any.whl

mlrun/projects/project.py

@@ -725,7 +725,7 @@ def _project_instance_from_struct(struct, name, allow_cross_project):
             # TODO: Remove this warning in version 1.9.0 and also fix cli to support allow_cross_project
             warnings.warn(
                 f"Project {name=} is different than specified on the context's project yaml. "
-                "This behavior is deprecated and will not be supported in version 1.9.0."
+                "This behavior is deprecated and will not be supported from version 1.9.0."
             )
             logger.warn(error_message)
         elif allow_cross_project:
@@ -3781,7 +3781,7 @@ class MlrunProject(ModelObj):
 
 
         :param name: Return only functions with a specific name.
-        :param tag: Return function versions with specific tags.
+        :param tag: Return function versions with specific tags. To return only tagged functions, set tag to ``"*"``.
         :param labels: Return functions that have specific labels assigned to them.
         :returns: List of function objects.
         """
@@ -4063,6 +4063,12 @@ class MlrunProject(ModelObj):
         db = mlrun.db.get_run_db(secrets=self._secrets)
         if alert_name is None:
             alert_name = alert_data.name
+        if alert_data.project is not None and alert_data.project != self.metadata.name:
+            logger.warn(
+                "Project in alert does not match project in operation",
+                project=alert_data.project,
+            )
+        alert_data.project = self.metadata.name
         return db.store_alert_config(alert_name, alert_data, project=self.metadata.name)
 
     def get_alert_config(self, alert_name: str) -> AlertConfig:

mlrun/run.py

@@ -21,6 +21,7 @@ import tempfile
 import time
 import typing
 import uuid
+import warnings
 from base64 import b64decode
 from copy import deepcopy
 from os import environ, makedirs, path
@@ -196,18 +197,19 @@ def load_func_code(command="", workdir=None, secrets=None, name="name"):
 def get_or_create_ctx(
     name: str,
     event=None,
-    spec=None,
+    spec: Optional[dict] = None,
     with_env: bool = True,
     rundb: str = "",
     project: str = "",
-    upload_artifacts=False,
+    upload_artifacts: bool = False,
     labels: Optional[dict] = None,
 ) -> MLClientCtx:
-    """called from within the user program to obtain a run context
+    """
+    Called from within the user program to obtain a run context.
 
-    the run context is an interface for receiving parameters, data and logging
+    The run context is an interface for receiving parameters, data and logging
     run results, the run context is read from the event, spec, or environment
-    (in that order), user can also work without a context (local defaults mode)
+    (in that order), user can also work without a context (local defaults mode).
 
     all results are automatically stored in the "rundb" or artifact store,
     the path to the rundb can be specified in the call or obtained from env.
@@ -220,7 +222,7 @@ def get_or_create_ctx(
     :param project:  project to initiate the context in (by default `mlrun.mlconf.default_project`)
     :param upload_artifacts:  when using local context (not as part of a job/run), upload artifacts to the
                               system default artifact path location
-    :param labels:   dict of the context labels
+    :param labels: (deprecated - use spec instead) dict of the context labels.
     :return: execution context
 
     Examples::
@@ -253,6 +255,20 @@ def get_or_create_ctx(
         context.log_artifact("results.html", body=b"<b> Some HTML <b>", viewer="web-app")
 
     """
+    if labels:
+        warnings.warn(
+            "The `labels` argument is deprecated and will be removed in 1.9.0. "
+            "Please use `spec` instead, e.g.:\n"
+            "spec={'metadata': {'labels': {'key': 'value'}}}",
+            FutureWarning,
+        )
+        if spec is None:
+            spec = {}
+        if "metadata" not in spec:
+            spec["metadata"] = {}
+        if "labels" not in spec["metadata"]:
+            spec["metadata"]["labels"] = {}
+        spec["metadata"]["labels"].update(labels)
 
     if global_context.get() and not spec and not event:
         return global_context.get()
@@ -306,9 +322,6 @@ def get_or_create_ctx(
     ctx = MLClientCtx.from_dict(
         newspec, rundb=out, autocommit=autocommit, tmp=tmp, host=socket.gethostname()
     )
-    labels = labels or {}
-    for key, val in labels.items():
-        ctx.set_label(key=key, value=val)
     global_context.set(ctx)
     return ctx
 

mlrun/runtimes/nuclio/api_gateway.py

@@ -326,6 +326,11 @@ class APIGatewaySpec(ModelObj):
         return function_names
 
 
+class APIGatewayStatus(ModelObj):
+    def __init__(self, state: Optional[schemas.APIGatewayState] = None):
+        self.state = state or schemas.APIGatewayState.none
+
+
 class APIGateway(ModelObj):
     _dict_fields = [
         "metadata",
@@ -338,16 +343,18 @@ class APIGateway(ModelObj):
         self,
         metadata: APIGatewayMetadata,
         spec: APIGatewaySpec,
+        status: Optional[APIGatewayStatus] = None,
     ):
         """
         Initialize the APIGateway instance.
 
         :param metadata: (APIGatewayMetadata) The metadata of the API gateway.
         :param spec: (APIGatewaySpec) The spec of the API gateway.
+        :param status: (APIGatewayStatus) The status of the API gateway.
         """
         self.metadata = metadata
         self.spec = spec
-        self.state = ""
+        self.status = status
 
     @property
     def metadata(self) -> APIGatewayMetadata:
@@ -365,6 +372,14 @@ class APIGateway(ModelObj):
     def spec(self, spec):
         self._spec = self._verify_dict(spec, "spec", APIGatewaySpec)
 
+    @property
+    def status(self) -> APIGatewayStatus:
+        return self._status
+
+    @status.setter
+    def status(self, status):
+        self._status = self._verify_dict(status, "status", APIGatewayStatus)
+
     def invoke(
         self,
         method="POST",
@@ -394,7 +409,7 @@ class APIGateway(ModelObj):
                 )
         if not self.is_ready():
             raise mlrun.errors.MLRunPreconditionFailedError(
-                f"API gateway is not ready. " f"Current state: {self.state}"
+                f"API gateway is not ready. " f"Current state: {self.status.state}"
             )
 
         auth = None
@@ -459,10 +474,10 @@ class APIGateway(ModelObj):
         )
 
     def is_ready(self):
-        if self.state is not schemas.api_gateway.APIGatewayState.ready:
+        if self.status.state is not schemas.api_gateway.APIGatewayState.ready:
             # try to sync the state
             self.sync()
-        return self.state == schemas.api_gateway.APIGatewayState.ready
+        return self.status.state == schemas.api_gateway.APIGatewayState.ready
 
     def sync(self):
         """
@@ -479,7 +494,7 @@ class APIGateway(ModelObj):
         self.spec.functions = synced_gateway.spec.functions
         self.spec.canary = synced_gateway.spec.canary
         self.spec.description = synced_gateway.spec.description
-        self.state = synced_gateway.state
+        self.status.state = synced_gateway.status.state
 
     def with_basic_auth(self, username: str, password: str):
         """
@@ -546,6 +561,14 @@ class APIGateway(ModelObj):
             project=self.spec.project, functions=self.spec.functions, ports=ports
         )
 
+    def with_force_ssl_redirect(self):
+        """
+        Set SSL redirect annotation for the API gateway.
+        """
+        self.metadata.annotations["nginx.ingress.kubernetes.io/force-ssl-redirect"] = (
+            "true"
+        )
+
     @classmethod
     def from_scheme(cls, api_gateway: schemas.APIGateway):
         project = api_gateway.metadata.labels.get(
@@ -560,6 +583,8 @@ class APIGateway(ModelObj):
         new_api_gateway = cls(
             metadata=APIGatewayMetadata(
                 name=api_gateway.spec.name,
+                annotations=api_gateway.metadata.annotations,
+                labels=api_gateway.metadata.labels,
             ),
             spec=APIGatewaySpec(
                 project=project,
@@ -570,8 +595,8 @@ class APIGateway(ModelObj):
                 functions=functions,
                 canary=canary,
             ),
+            status=APIGatewayStatus(state=state),
         )
-        new_api_gateway.state = state
         return new_api_gateway
 
     def to_scheme(self) -> schemas.APIGateway:
@@ -600,7 +625,11 @@ class APIGateway(ModelObj):
                 upstreams[i].port = port
 
         api_gateway = schemas.APIGateway(
-            metadata=schemas.APIGatewayMetadata(name=self.metadata.name, labels={}),
+            metadata=schemas.APIGatewayMetadata(
+                name=self.metadata.name,
+                labels=self.metadata.labels,
+                annotations=self.metadata.annotations,
+            ),
             spec=schemas.APIGatewaySpec(
                 name=self.metadata.name,
                 description=self.spec.description,
@@ -611,6 +640,7 @@ class APIGateway(ModelObj):
                 ),
                 upstreams=upstreams,
             ),
+            status=schemas.APIGatewayStatus(state=self.status.state),
         )
         api_gateway.spec.authentication = self.spec.authentication.to_scheme()
         return api_gateway

mlrun/runtimes/nuclio/application/application.py

@@ -149,6 +149,7 @@ class ApplicationStatus(NuclioStatus):
         build_pod=None,
         container_image=None,
         application_image=None,
+        application_source=None,
         sidecar_name=None,
         api_gateway_name=None,
         api_gateway=None,
@@ -164,6 +165,7 @@ class ApplicationStatus(NuclioStatus):
             container_image=container_image,
         )
         self.application_image = application_image or None
+        self.application_source = application_source or None
         self.sidecar_name = sidecar_name or None
         self.api_gateway_name = api_gateway_name or None
         self.api_gateway = api_gateway or None
@@ -266,6 +268,7 @@ class ApplicationRuntime(RemoteRuntime):
         direct_port_access: bool = False,
         authentication_mode: schemas.APIGatewayAuthenticationMode = None,
         authentication_creds: tuple[str] = None,
+        ssl_redirect: bool = None,
     ):
         """
         Deploy function, builds the application image if required (self.requires_build()) or force_build is True,
@@ -285,6 +288,9 @@ class ApplicationRuntime(RemoteRuntime):
         :param direct_port_access:      Set True to allow direct port access to the application sidecar
         :param authentication_mode:     API Gateway authentication mode
         :param authentication_creds:    API Gateway authentication credentials as a tuple (username, password)
+        :param ssl_redirect:            Set True to force SSL redirect, False to disable. Defaults to
+                                        mlrun.mlconf.force_api_gateway_ssl_redirect()
+
         :return: True if the function is ready (deployed)
         """
         if self.requires_build() or force_build:
@@ -313,19 +319,30 @@ class ApplicationRuntime(RemoteRuntime):
         )
 
         super().deploy(
-            project,
-            tag,
-            verbose,
-            auth_info,
-            builder_env,
+            project=project,
+            tag=tag,
+            verbose=verbose,
+            auth_info=auth_info,
+            builder_env=builder_env,
+        )
+        logger.info(
+            "Successfully deployed function, creating API gateway",
+            api_gateway_name=self.status.api_gateway_name,
+            authentication_mode=authentication_mode,
         )
 
+        # Restore the source in case it was removed to make nuclio not consider it when building
+        if not self.spec.build.source and self.status.application_source:
+            self.spec.build.source = self.status.application_source
+        self.save(versioned=False)
+
         ports = self.spec.internal_application_port if direct_port_access else []
-        self.create_api_gateway(
+        return self.create_api_gateway(
             name=self.status.api_gateway_name,
             ports=ports,
             authentication_mode=authentication_mode,
             authentication_creds=authentication_creds,
+            ssl_redirect=ssl_redirect,
         )
 
     def with_source_archive(
@@ -349,6 +366,14 @@ class ApplicationRuntime(RemoteRuntime):
             target_dir=target_dir,
         )
 
+    def from_image(self, image):
+        super().from_image(image)
+        # nuclio implementation detail - when providing the image and emptying out the source code and build source,
+        # nuclio skips rebuilding the image and simply takes the prebuilt image
+        self.spec.build.functionSourceCode = ""
+        self.status.application_source = self.spec.build.source
+        self.spec.build.source = ""
+
     @classmethod
     def get_filename_and_handler(cls) -> (str, str):
         reverse_proxy_file_path = pathlib.Path(__file__).parent / "reverse_proxy.go"
@@ -361,6 +386,7 @@ class ApplicationRuntime(RemoteRuntime):
         ports: list[int] = None,
         authentication_mode: schemas.APIGatewayAuthenticationMode = None,
         authentication_creds: tuple[str] = None,
+        ssl_redirect: bool = None,
     ):
         api_gateway = APIGateway(
             APIGatewayMetadata(
@@ -377,6 +403,13 @@ class ApplicationRuntime(RemoteRuntime):
             ),
         )
 
+        if ssl_redirect is None:
+            ssl_redirect = mlrun.mlconf.force_api_gateway_ssl_redirect()
+        if ssl_redirect:
+            # force ssl redirect so that the application is only accessible via https
+            api_gateway.with_force_ssl_redirect()
+
+        # add authentication if required
         authentication_mode = (
             authentication_mode
             or mlrun.mlconf.function.application.default_authentication_mode
@@ -396,6 +429,9 @@ class ApplicationRuntime(RemoteRuntime):
         self.status.api_gateway.wait_for_readiness()
         self.url = self.status.api_gateway.invoke_url
 
+        logger.info("Successfully created API gateway", url=self.url)
+        return self.url
+
     def invoke(
         self,
         path: str,
@@ -435,6 +471,14 @@ class ApplicationRuntime(RemoteRuntime):
             **http_client_kwargs,
         )
 
+    def _run(self, runobj: "mlrun.RunObject", execution):
+        raise mlrun.runtimes.RunError(
+            "Application runtime .run() is not yet supported. Use .invoke() instead."
+        )
+
+    def _enrich_command_from_status(self):
+        pass
+
     def _build_application_image(
         self,
         builder_env: dict = None,
@@ -493,9 +537,6 @@ class ApplicationRuntime(RemoteRuntime):
 
         if self.status.container_image:
             self.from_image(self.status.container_image)
-            # nuclio implementation detail - when providing the image and emptying out the source code,
-            # nuclio skips rebuilding the image and simply takes the prebuilt image
-            self.spec.build.functionSourceCode = ""
 
         self.status.sidecar_name = f"{self.metadata.name}-sidecar"
         self.with_sidecar(

mlrun/runtimes/nuclio/function.py

@@ -568,6 +568,9 @@ class RemoteRuntime(KubeResource):
         # this also means that the function object will be updated with the function status
         self._wait_for_function_deployment(db, verbose=verbose)
 
+        return self._enrich_command_from_status()
+
+    def _enrich_command_from_status(self):
         # NOTE: on older mlrun versions & nuclio versions, function are exposed via NodePort
         #       now, functions can be not exposed (using service type ClusterIP) and hence
         #       for BC we first try to populate the external invocation url, and then

mlrun/runtimes/nuclio/serving.py

@@ -325,12 +325,12 @@ class ServingRuntime(RemoteRuntime):
         :param enable_tracking:     Enabled/Disable model-monitoring tracking.
                                     Default True (tracking enabled).
 
-                                example::
+        Example::
 
-                                    # initialize a new serving function
-                                    serving_fn = mlrun.import_function("hub://v2-model-server", new_name="serving")
-                                    # apply model monitoring
-                                    serving_fn.set_tracking()
+            # initialize a new serving function
+            serving_fn = mlrun.import_function("hub://v2-model-server", new_name="serving")
+            # apply model monitoring
+            serving_fn.set_tracking()
 
         """
         # Applying model monitoring configurations

mlrun/serving/server.py

@@ -321,9 +321,9 @@ def v2_serving_init(context, namespace=None):
         server.http_trigger = getattr(context.trigger, "kind", "http") == "http"
     context.logger.info_with(
         "Setting current function",
-        current_functiton=os.environ.get("SERVING_CURRENT_FUNCTION", ""),
+        current_function=os.getenv("SERVING_CURRENT_FUNCTION", ""),
     )
-    server.set_current_function(os.environ.get("SERVING_CURRENT_FUNCTION", ""))
+    server.set_current_function(os.getenv("SERVING_CURRENT_FUNCTION", ""))
     context.logger.info_with(
         "Initializing states", namespace=namespace or get_caller_globals()
     )
@@ -344,9 +344,14 @@ def v2_serving_init(context, namespace=None):
     if server.verbose:
         context.logger.info(server.to_yaml())
 
-    if hasattr(context, "platform") and hasattr(
-        context.platform, "set_termination_callback"
-    ):
+    _set_callbacks(server, context)
+
+
+def _set_callbacks(server, context):
+    if not server.graph.supports_termination() or not hasattr(context, "platform"):
+        return
+
+    if hasattr(context.platform, "set_termination_callback"):
         context.logger.info(
             "Setting termination callback to terminate graph on worker shutdown"
         )
@@ -358,7 +363,7 @@ def v2_serving_init(context, namespace=None):
 
         context.platform.set_termination_callback(termination_callback)
 
-    if hasattr(context, "platform") and hasattr(context.platform, "set_drain_callback"):
+    if hasattr(context.platform, "set_drain_callback"):
         context.logger.info(
             "Setting drain callback to terminate and restart the graph on a drain event (such as rebalancing)"
         )
@@ -417,7 +422,7 @@ def create_graph_server(
     parameters = parameters or {}
     server = GraphServer(graph, parameters, load_mode, verbose=verbose, **kwargs)
     server.set_current_function(
-        current_function or os.environ.get("SERVING_CURRENT_FUNCTION", "")
+        current_function or os.getenv("SERVING_CURRENT_FUNCTION", "")
     )
     return server
 

mlrun/serving/states.py

@@ -27,6 +27,8 @@ from copy import copy, deepcopy
 from inspect import getfullargspec, signature
 from typing import Any, Union
 
+import storey.utils
+
 import mlrun
 
 from ..config import config
@@ -386,6 +388,9 @@ class BaseStep(ModelObj):
         """
         raise NotImplementedError("set_flow() can only be called on a FlowStep")
 
+    def supports_termination(self):
+        return False
+
 
 class TaskStep(BaseStep):
     """task execution step, runs a class or handler"""
@@ -867,7 +872,9 @@ class QueueStep(BaseStep):
             return event
 
         if self._stream:
-            self._stream.push({"id": event.id, "body": data, "path": event.path})
+            if self.options.get("full_event", True):
+                data = storey.utils.wrap_event_for_serialization(event, data)
+            self._stream.push(data)
             event.terminated = True
             event.body = None
         return event
@@ -1273,6 +1280,8 @@ class FlowStep(BaseStep):
             event.body = {"id": event.id}
             return event
 
+        event = storey.utils.unpack_event_if_wrapped(event)
+
         if len(self._start_steps) == 0:
             return event
         next_obj = self._start_steps[0]
@@ -1380,6 +1389,9 @@ class FlowStep(BaseStep):
 
         return step
 
+    def supports_termination(self):
+        return self.engine == "async"
+
 
 class RootFlowStep(FlowStep):
     """root flow step"""

mlrun/utils/db.py

@@ -28,6 +28,9 @@ class BaseModel:
         columns = [column.key for column in mapper.columns if column.key not in exclude]
 
         def get_key_value(c):
+            # all (never say never) DB classes have "object" defined as "full_object"
+            if c == "object":
+                c = "full_object"
             if isinstance(getattr(self, c), datetime):
                 return c, getattr(self, c).isoformat()
             return c, getattr(self, c)

mlrun/utils/helpers.py

@@ -111,13 +111,11 @@ def get_artifact_target(item: dict, project=None):
     tree = item["metadata"].get("tree")
     tag = item["metadata"].get("tag")
 
-    kind = item.get("kind")
-    if kind in ["dataset", "model", "artifact"] and db_key:
+    if item.get("kind") in {"dataset", "model", "artifact"} and db_key:
         target = f"{DB_SCHEMA}://{StorePrefix.Artifact}/{project_str}/{db_key}"
-        if tag:
-            target = f"{target}:{tag}"
+        target += f":{tag}" if tag else ":latest"
         if tree:
-            target = f"{target}@{tree}"
+            target += f"@{tree}"
         return target
 
     return item["spec"].get("target_path")

mlrun/utils/notifications/notification/webhook.py

@@ -60,7 +60,14 @@ class WebhookNotification(NotificationBase):
             request_body["runs"] = runs
 
         if alert:
-            request_body["alert"] = alert.dict()
+            request_body["name"] = alert.name
+            request_body["project"] = alert.project
+            request_body["severity"] = alert.severity
+            if alert.summary:
+                request_body["summary"] = mlrun.utils.helpers.format_alert_summary(
+                    alert, event_data
+                )
+
             if event_data:
                 request_body["value"] = event_data.value_dict
                 request_body["id"] = event_data.entity.ids[0]

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "8180ea5cac9bb9ed6be04b85fd1ba9a79f3a9f9a",
-  "version": "1.7.0-rc32"
+  "git_commit": "4542af29f22f596b4e09be897f728a6e8f676b55",
+  "version": "1.7.0-rc34"
 }

{mlrun-1.7.0rc32.dist-info → mlrun-1.7.0rc34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.7.0rc32
+Version: 1.7.0rc34
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -43,10 +43,10 @@ Requires-Dist: semver ~=3.0
 Requires-Dist: dependency-injector ~=4.41
 Requires-Dist: fsspec <2024.4,>=2023.9.2
 Requires-Dist: v3iofs ~=0.1.17
-Requires-Dist: storey ~=1.7.20
+Requires-Dist: storey ~=1.7.23
 Requires-Dist: inflection ~=0.5.0
 Requires-Dist: python-dotenv ~=0.17.0
-Requires-Dist: setuptools ~=70.0
+Requires-Dist: setuptools ~=71.0
 Requires-Dist: deprecated ~=1.2
 Requires-Dist: jinja2 >=3.1.3,~=3.1
 Requires-Dist: orjson <4,>=3.9.15