mlrun 1.7.0rc15__py3-none-any.whl → 1.7.0rc17__py3-none-any.whl

mlrun/run.py

@@ -29,11 +29,13 @@ from typing import Optional, Union
 import nuclio
 import yaml
 from kfp import Client
+from mlrun_pipelines.common.models import RunStatuses
+from mlrun_pipelines.common.ops import format_summary_from_kfp_run, show_kfp_run
+from mlrun_pipelines.models import PipelineRun
 
 import mlrun.common.schemas
 import mlrun.errors
 import mlrun.utils.helpers
-from mlrun.kfpops import format_summary_from_kfp_run, show_kfp_run
 
 from .common.helpers import parse_versioned_object_uri
 from .config import config as mlconf
@@ -68,41 +70,6 @@ from .utils import (
 )
 
 
-class RunStatuses:
-    succeeded = "Succeeded"
-    failed = "Failed"
-    skipped = "Skipped"
-    error = "Error"
-    running = "Running"
-
-    @staticmethod
-    def all():
-        return [
-            RunStatuses.succeeded,
-            RunStatuses.failed,
-            RunStatuses.skipped,
-            RunStatuses.error,
-            RunStatuses.running,
-        ]
-
-    @staticmethod
-    def stable_statuses():
-        return [
-            RunStatuses.succeeded,
-            RunStatuses.failed,
-            RunStatuses.skipped,
-            RunStatuses.error,
-        ]
-
-    @staticmethod
-    def transient_statuses():
-        return [
-            status
-            for status in RunStatuses.all()
-            if status not in RunStatuses.stable_statuses()
-        ]
-
-
 def function_to_module(code="", workdir=None, secrets=None, silent=False):
     """Load code, notebook or mlrun function as .py module
     this function can import a local/remote py file or notebook
@@ -1021,7 +988,7 @@ def get_pipeline(
     :param project:    the project of the pipeline run
     :param remote:     read kfp data from mlrun service (default=True)
 
-    :return: kfp run dict
+    :return: kfp run
     """
     namespace = namespace or mlconf.namespace
     if remote:
@@ -1045,7 +1012,7 @@ def get_pipeline(
                 not format_
                 or format_ == mlrun.common.schemas.PipelinesFormat.summary.value
             ):
-                resp = format_summary_from_kfp_run(resp)
+                resp = format_summary_from_kfp_run(PipelineRun(resp))
 
     show_kfp_run(resp)
     return resp

mlrun/runtimes/__init__.py

@@ -43,6 +43,7 @@ from .nuclio import (
     new_v2_model_server,
     nuclio_init_hook,
 )
+from .nuclio.api_gateway import APIGateway
 from .nuclio.application import ApplicationRuntime
 from .nuclio.serving import serving_subkind
 from .remotesparkjob import RemoteSparkRuntime

mlrun/runtimes/base.py

@@ -21,6 +21,7 @@ from os import environ
 from typing import Callable, Optional, Union
 
 import requests.exceptions
+from mlrun_pipelines.common.ops import mlrun_op
 from nuclio.build import mlrun_footer
 
 import mlrun.common.constants
@@ -37,7 +38,6 @@ from mlrun.utils.helpers import generate_object_uri, verify_field_regex
 from ..config import config
 from ..datastore import store_manager
 from ..errors import err_to_str
-from ..kfpops import mlrun_op
 from ..lists import RunList
 from ..model import BaseMetadata, HyperParamOptions, ImageBuilder, ModelObj, RunObject
 from ..utils import (
@@ -707,11 +707,11 @@ class BaseRuntime(ModelObj):
                                   "key": "the_key".
         :param auto_build:      when set to True and the function require build it will be built on the first
                                 function run, use only if you dont plan on changing the build config between runs
-        :return: KubeFlow containerOp
+        :return: mlrun_pipelines.models.PipelineNodeWrapper
         """
 
         # if the function contain KFP PipelineParams (futures) pass the full spec to the
-        # ContainerOp this way KFP will substitute the params with previous step outputs
+        # PipelineNodeWrapper this way KFP will substitute the params with previous step outputs
         if use_db and not self._has_pipeline_param():
             # if the same function is built as part of the pipeline we do not use the versioned function
             # rather the latest function w the same tag so we can pick up the updated image/status

mlrun/runtimes/kubejob.py

@@ -14,11 +14,12 @@
 
 import warnings
 
+from mlrun_pipelines.common.ops import build_op
+
 import mlrun.common.schemas
 import mlrun.db
 import mlrun.errors
 
-from ..kfpops import build_op
 from ..model import RunObject
 from .pod import KubeResource
 

mlrun/runtimes/nuclio/api_gateway.py

@@ -17,31 +17,31 @@ from typing import Optional, Union
 from urllib.parse import urljoin
 
 import requests
-from requests.auth import HTTPBasicAuth
+from nuclio.auth import AuthInfo as NuclioAuthInfo
+from nuclio.auth import AuthKinds as NuclioAuthKinds
 
 import mlrun
-import mlrun.common.schemas
+import mlrun.common.schemas as schemas
+import mlrun.common.types
+from mlrun.model import ModelObj
+from mlrun.platforms.iguazio import min_iguazio_versions
+from mlrun.utils import logger
 
-from ...model import ModelObj
-from ..utils import logger
-from .function import RemoteRuntime, get_fullname, min_nuclio_versions
-from .serving import ServingRuntime
+from .function import get_fullname, min_nuclio_versions
 
-NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH = "basicAuth"
-NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_NONE = "none"
 PROJECT_NAME_LABEL = "nuclio.io/project-name"
 
 
 class APIGatewayAuthenticator(typing.Protocol):
     @property
     def authentication_mode(self) -> str:
-        return NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_NONE
+        return schemas.APIGatewayAuthenticationMode.none.value
 
     @classmethod
-    def from_scheme(cls, api_gateway_spec: mlrun.common.schemas.APIGatewaySpec):
+    def from_scheme(cls, api_gateway_spec: schemas.APIGatewaySpec):
         if (
             api_gateway_spec.authenticationMode
-            == NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH
+            == schemas.APIGatewayAuthenticationMode.basic.value
         ):
             if api_gateway_spec.authentication:
                 return BasicAuth(
@@ -50,12 +50,17 @@ class APIGatewayAuthenticator(typing.Protocol):
                 )
             else:
                 return BasicAuth()
+        elif (
+            api_gateway_spec.authenticationMode
+            == schemas.APIGatewayAuthenticationMode.access_key.value
+        ):
+            return AccessKeyAuth()
         else:
             return NoneAuth()
 
     def to_scheme(
         self,
-    ) -> Optional[dict[str, Optional[mlrun.common.schemas.APIGatewayBasicAuth]]]:
+    ) -> Optional[dict[str, Optional[schemas.APIGatewayBasicAuth]]]:
         return None
 
 
@@ -81,18 +86,28 @@ class BasicAuth(APIGatewayAuthenticator):
 
     @property
     def authentication_mode(self) -> str:
-        return NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH
+        return schemas.APIGatewayAuthenticationMode.basic.value
 
     def to_scheme(
         self,
-    ) -> Optional[dict[str, Optional[mlrun.common.schemas.APIGatewayBasicAuth]]]:
+    ) -> Optional[dict[str, Optional[schemas.APIGatewayBasicAuth]]]:
         return {
-            "basicAuth": mlrun.common.schemas.APIGatewayBasicAuth(
+            "basicAuth": schemas.APIGatewayBasicAuth(
                 username=self._username, password=self._password
             )
         }
 
 
+class AccessKeyAuth(APIGatewayAuthenticator):
+    """
+    An API gateway authenticator with access key authentication.
+    """
+
+    @property
+    def authentication_mode(self) -> str:
+        return schemas.APIGatewayAuthenticationMode.access_key.value
+
+
 class APIGatewayMetadata(ModelObj):
     _dict_fields = ["name", "namespace", "labels", "annotations", "creation_timestamp"]
 
@@ -138,17 +153,17 @@ class APIGatewaySpec(ModelObj):
     def __init__(
         self,
         functions: Union[
-            list[str],
-            Union[
-                list[
-                    Union[
-                        RemoteRuntime,
-                        ServingRuntime,
-                    ]
-                ],
-                RemoteRuntime,
-                ServingRuntime,
+            list[
+                Union[
+                    str,
+                    "mlrun.runtimes.nuclio.function.RemoteRuntime",
+                    "mlrun.runtimes.nuclio.serving.ServingRuntime",
+                    "mlrun.runtimes.nuclio.application.ApplicationRuntime",
+                ]
             ],
+            "mlrun.runtimes.nuclio.function.RemoteRuntime",
+            "mlrun.runtimes.nuclio.serving.ServingRuntime",
+            "mlrun.runtimes.nuclio.application.ApplicationRuntime",
         ],
         project: str = None,
         description: str = "",
@@ -156,20 +171,24 @@ class APIGatewaySpec(ModelObj):
         path: str = "/",
         authentication: Optional[APIGatewayAuthenticator] = NoneAuth(),
         canary: Optional[list[int]] = None,
+        ports: Optional[list[int]] = None,
     ):
         """
         :param functions: The list of functions associated with the API gateway
             Can be a list of function names (["my-func1", "my-func2"])
             or a list or a single entity of
             :py:class:`~mlrun.runtimes.nuclio.function.RemoteRuntime` OR
-            :py:class:`~mlrun.runtimes.nuclio.serving.ServingRuntime`
+            :py:class:`~mlrun.runtimes.nuclio.serving.ServingRuntime` OR
+            :py:class:`~mlrun.runtimes.nuclio.application.ApplicationRuntime`
         :param project: The project name
         :param description: Optional description of the API gateway
         :param path: Optional path of the API gateway, default value is "/"
         :param authentication: The authentication for the API gateway of type
                 :py:class:`~mlrun.runtimes.nuclio.api_gateway.BasicAuth`
         :param host:  The host of the API gateway (optional). If not set, it will be automatically generated
-        :param canary: The canary percents for the API gateway of type list[int]; for instance: [20,80]
+        :param canary: The canary percents for the API gateway of type list[int]; for instance: [20,80] (optional)
+        :param ports: The ports of the API gateway, as a list of integers that correspond to the functions in the
+            functions list. for instance: [8050] or [8050, 8081] (optional)
         """
         self.description = description
         self.host = host
@@ -178,26 +197,28 @@ class APIGatewaySpec(ModelObj):
         self.functions = functions
         self.canary = canary
         self.project = project
+        self.ports = ports
 
-        self.validate(project=project, functions=functions, canary=canary)
+        self.validate(project=project, functions=functions, canary=canary, ports=ports)
 
     def validate(
         self,
         project: str,
         functions: Union[
-            list[str],
-            Union[
-                list[
-                    Union[
-                        RemoteRuntime,
-                        ServingRuntime,
-                    ]
-                ],
-                RemoteRuntime,
-                ServingRuntime,
+            list[
+                Union[
+                    str,
+                    "mlrun.runtimes.nuclio.function.RemoteRuntime",
+                    "mlrun.runtimes.nuclio.serving.ServingRuntime",
+                    "mlrun.runtimes.nuclio.application.ApplicationRuntime",
+                ]
             ],
+            "mlrun.runtimes.nuclio.function.RemoteRuntime",
+            "mlrun.runtimes.nuclio.serving.ServingRuntime",
+            "mlrun.runtimes.nuclio.application.ApplicationRuntime",
         ],
         canary: Optional[list[int]] = None,
+        ports: Optional[list[int]] = None,
     ):
         self.functions = self._validate_functions(project=project, functions=functions)
 
@@ -205,6 +226,10 @@ class APIGatewaySpec(ModelObj):
         if canary:
             self.canary = self._validate_canary(canary)
 
+        # validating ports
+        if ports:
+            self.ports = self._validate_ports(ports)
+
     def _validate_canary(self, canary: list[int]):
         if len(self.functions) != len(canary):
             raise mlrun.errors.MLRunInvalidArgumentError(
@@ -221,20 +246,29 @@ class APIGatewaySpec(ModelObj):
             )
         return canary
 
+    def _validate_ports(self, ports):
+        if len(self.functions) != len(ports):
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Function and port lists lengths do not match"
+            )
+
+        return ports
+
     @staticmethod
     def _validate_functions(
         project: str,
         functions: Union[
-            list[str],
-            Union[
-                list[
-                    Union[
-                        RemoteRuntime,
-                        ServingRuntime,
-                    ]
-                ],
-                Union[RemoteRuntime, ServingRuntime],
+            list[
+                Union[
+                    str,
+                    "mlrun.runtimes.nuclio.function.RemoteRuntime",
+                    "mlrun.runtimes.nuclio.serving.ServingRuntime",
+                    "mlrun.runtimes.nuclio.application.ApplicationRuntime",
+                ]
             ],
+            "mlrun.runtimes.nuclio.function.RemoteRuntime",
+            "mlrun.runtimes.nuclio.serving.ServingRuntime",
+            "mlrun.runtimes.nuclio.application.ApplicationRuntime",
         ],
     ):
         if not isinstance(functions, list):
@@ -312,8 +346,9 @@ class APIGateway(ModelObj):
     def invoke(
         self,
         method="POST",
-        headers: dict = {},
-        auth: Optional[tuple[str, str]] = None,
+        headers: dict = None,
+        credentials: Optional[tuple[str, str]] = None,
+        path: Optional[str] = None,
         **kwargs,
     ):
         """
@@ -321,7 +356,9 @@ class APIGateway(ModelObj):
 
         :param method: (str, optional) The HTTP method for the invocation.
         :param headers: (dict, optional) The HTTP headers for the invocation.
-        :param auth: (Optional[tuple[str, str]], optional) The authentication creds for the invocation if required.
+        :param credentials: (Optional[tuple[str, str]], optional) The (username,password) for the invocation if required
+            can also be set by the environment variable (_, V3IO_ACCESS_KEY) for access key authentication.
+        :param path: (str, optional) The sub-path for the invocation.
         :param kwargs: (dict) Additional keyword arguments.
 
         :return: The response from the API gateway invocation.
@@ -338,20 +375,44 @@ class APIGateway(ModelObj):
                 f"API gateway is not ready. " f"Current state: {self.state}"
             )
 
+        auth = None
+
         if (
             self.spec.authentication.authentication_mode
-            == NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH
-            and not auth
+            == schemas.APIGatewayAuthenticationMode.basic.value
         ):
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "API Gateway invocation requires authentication. Please pass credentials"
-            )
+            if not credentials:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "API Gateway invocation requires authentication. Please pass credentials"
+                )
+            auth = NuclioAuthInfo(
+                username=credentials[0], password=credentials[1]
+            ).to_requests_auth()
+
+        if (
+            self.spec.authentication.authentication_mode
+            == schemas.APIGatewayAuthenticationMode.access_key.value
+        ):
+            # inject access key from env
+            if credentials:
+                auth = NuclioAuthInfo(
+                    username=credentials[0],
+                    password=credentials[1],
+                    mode=NuclioAuthKinds.iguazio,
+                ).to_requests_auth()
+            else:
+                auth = NuclioAuthInfo().from_envvar().to_requests_auth()
+            if not auth:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "API Gateway invocation requires authentication. Please set V3IO_ACCESS_KEY env var"
+                )
+        url = urljoin(self.invoke_url, path or "")
         return requests.request(
             method=method,
-            url=self.invoke_url,
-            headers=headers,
+            url=url,
+            headers=headers or {},
+            auth=auth,
             **kwargs,
-            auth=HTTPBasicAuth(*auth) if auth else None,
         )
 
     def wait_for_readiness(self, max_wait_time=90):
@@ -376,10 +437,10 @@ class APIGateway(ModelObj):
         )
 
     def is_ready(self):
-        if self.state is not mlrun.common.schemas.api_gateway.APIGatewayState.ready:
+        if self.state is not schemas.api_gateway.APIGatewayState.ready:
             # try to sync the state
             self.sync()
-        return self.state == mlrun.common.schemas.api_gateway.APIGatewayState.ready
+        return self.state == schemas.api_gateway.APIGatewayState.ready
 
     def sync(self):
         """
@@ -407,16 +468,27 @@ class APIGateway(ModelObj):
         """
         self.spec.authentication = BasicAuth(username=username, password=password)
 
+    @min_iguazio_versions("3.5.5")
+    def with_access_key_auth(self):
+        """
+        Set access key authentication for the API gateway.
+        """
+        self.spec.authentication = AccessKeyAuth()
+
     def with_canary(
         self,
         functions: Union[
-            list[str],
             list[
                 Union[
-                    RemoteRuntime,
-                    ServingRuntime,
+                    str,
+                    "mlrun.runtimes.nuclio.function.RemoteRuntime",
+                    "mlrun.runtimes.nuclio.serving.ServingRuntime",
+                    "mlrun.runtimes.nuclio.application.ApplicationRuntime",
                 ]
             ],
+            "mlrun.runtimes.nuclio.function.RemoteRuntime",
+            "mlrun.runtimes.nuclio.serving.ServingRuntime",
+            "mlrun.runtimes.nuclio.application.ApplicationRuntime",
         ],
         canary: list[int],
     ):
@@ -427,7 +499,8 @@ class APIGateway(ModelObj):
             Can be a list of function names (["my-func1", "my-func2"])
             or a list of nuclio functions of types
             :py:class:`~mlrun.runtimes.nuclio.function.RemoteRuntime` OR
-            :py:class:`~mlrun.runtimes.nuclio.serving.ServingRuntime`
+            :py:class:`~mlrun.runtimes.nuclio.serving.ServingRuntime` OR
+            :py:class:`~mlrun.runtimes.nuclio.application.ApplicationRuntime`
         :param canary: The canary percents for the API gateway of type list[int]; for instance: [20,80]
 
         """
@@ -440,14 +513,25 @@ class APIGateway(ModelObj):
             project=self.spec.project, functions=functions, canary=canary
         )
 
+    def with_ports(self, ports: list[int]):
+        """
+        Set ports for the API gateway
+
+        :param ports: The ports of the API gateway, as a list of integers that correspond to the functions in the
+            functions list. for instance: [8050] or [8050, 8081]
+        """
+        self.spec.validate(
+            project=self.spec.project, functions=self.spec.functions, ports=ports
+        )
+
     @classmethod
-    def from_scheme(cls, api_gateway: mlrun.common.schemas.APIGateway):
+    def from_scheme(cls, api_gateway: schemas.APIGateway):
         project = api_gateway.metadata.labels.get(PROJECT_NAME_LABEL)
         functions, canary = cls._resolve_canary(api_gateway.spec.upstreams)
         state = (
             api_gateway.status.state
             if api_gateway.status
-            else mlrun.common.schemas.APIGatewayState.none
+            else schemas.APIGatewayState.none
         )
         new_api_gateway = cls(
             metadata=APIGatewayMetadata(
@@ -466,14 +550,14 @@ class APIGateway(ModelObj):
         new_api_gateway.state = state
         return new_api_gateway
 
-    def to_scheme(self) -> mlrun.common.schemas.APIGateway:
+    def to_scheme(self) -> schemas.APIGateway:
         upstreams = (
             [
-                mlrun.common.schemas.APIGatewayUpstream(
+                schemas.APIGatewayUpstream(
                     nucliofunction={"name": self.spec.functions[0]},
                     percentage=self.spec.canary[0],
                 ),
-                mlrun.common.schemas.APIGatewayUpstream(
+                schemas.APIGatewayUpstream(
                     # do not set percent for the second function,
                     # so we can define which function to display as a primary one in UI
                     nucliofunction={"name": self.spec.functions[1]},
@@ -481,22 +565,24 @@ class APIGateway(ModelObj):
             ]
             if self.spec.canary
             else [
-                mlrun.common.schemas.APIGatewayUpstream(
+                schemas.APIGatewayUpstream(
                     nucliofunction={"name": function_name},
                 )
                 for function_name in self.spec.functions
             ]
         )
-        api_gateway = mlrun.common.schemas.APIGateway(
-            metadata=mlrun.common.schemas.APIGatewayMetadata(
-                name=self.metadata.name, labels={}
-            ),
-            spec=mlrun.common.schemas.APIGatewaySpec(
+        if self.spec.ports:
+            for i, port in enumerate(self.spec.ports):
+                upstreams[i].port = port
+
+        api_gateway = schemas.APIGateway(
+            metadata=schemas.APIGatewayMetadata(name=self.metadata.name, labels={}),
+            spec=schemas.APIGatewaySpec(
                 name=self.metadata.name,
                 description=self.spec.description,
                 host=self.spec.host,
                 path=self.spec.path,
-                authenticationMode=mlrun.common.schemas.APIGatewayAuthenticationMode.from_str(
+                authenticationMode=schemas.APIGatewayAuthenticationMode.from_str(
                     self.spec.authentication.authentication_mode
                 ),
                 upstreams=upstreams,
@@ -526,7 +612,7 @@ class APIGateway(ModelObj):
 
     @staticmethod
     def _resolve_canary(
-        upstreams: list[mlrun.common.schemas.APIGatewayUpstream],
+        upstreams: list[schemas.APIGatewayUpstream],
     ) -> tuple[Union[list[str], None], Union[list[int], None]]:
         if len(upstreams) == 1:
             return [upstreams[0].nucliofunction.get("name")], None