mlrun 1.7.0rc12__py3-none-any.whl → 1.7.0rc14__py3-none-any.whl

mlrun/runtimes/nuclio/api_gateway.py

@@ -22,6 +22,7 @@ from requests.auth import HTTPBasicAuth
 import mlrun
 import mlrun.common.schemas
 
+from ...model import ModelObj
 from ..utils import logger
 from .function import RemoteRuntime, get_fullname, min_nuclio_versions
 from .serving import ServingRuntime
@@ -92,12 +93,50 @@ class BasicAuth(APIGatewayAuthenticator):
         }
 
 
-class APIGateway:
-    @min_nuclio_versions("1.13.1")
+class APIGatewayMetadata(ModelObj):
+    _dict_fields = ["name", "namespace", "labels", "annotations", "creation_timestamp"]
+
     def __init__(
         self,
-        project,
         name: str,
+        namespace: str = None,
+        labels: dict = None,
+        annotations: dict = None,
+        creation_timestamp: str = None,
+    ):
+        """
+        :param name: The name of the API gateway
+        :param namespace: The namespace of the API gateway
+        :param labels: The labels of the API gateway
+        :param annotations: The annotations of the API gateway
+        :param creation_timestamp: The creation timestamp of the API gateway
+        """
+        self.name = name
+        self.namespace = namespace
+        self.labels = labels or {}
+        self.annotations = annotations or {}
+        self.creation_timestamp = creation_timestamp
+
+        if not self.name:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "API Gateway name cannot be empty"
+            )
+
+
+class APIGatewaySpec(ModelObj):
+    _dict_fields = [
+        "functions",
+        "project",
+        "name",
+        "description",
+        "host",
+        "path",
+        "authentication",
+        "canary",
+    ]
+
+    def __init__(
+        self,
         functions: Union[
             list[str],
             Union[
@@ -107,26 +146,24 @@ class APIGateway:
                         ServingRuntime,
                     ]
                 ],
-                Union[RemoteRuntime, ServingRuntime],
+                RemoteRuntime,
+                ServingRuntime,
             ],
         ],
+        project: str = None,
         description: str = "",
+        host: str = None,
         path: str = "/",
         authentication: Optional[APIGatewayAuthenticator] = NoneAuth(),
-        host: Optional[str] = None,
         canary: Optional[list[int]] = None,
     ):
         """
-        Initialize the APIGateway instance.
-
-        :param project: The project name
-        :param name: The name of the API gateway
         :param functions: The list of functions associated with the API gateway
             Can be a list of function names (["my-func1", "my-func2"])
             or a list or a single entity of
             :py:class:`~mlrun.runtimes.nuclio.function.RemoteRuntime` OR
             :py:class:`~mlrun.runtimes.nuclio.serving.ServingRuntime`
-
+        :param project: The project name
         :param description: Optional description of the API gateway
         :param path: Optional path of the API gateway, default value is "/"
         :param authentication: The authentication for the API gateway of type
@@ -134,23 +171,144 @@ class APIGateway:
         :param host:  The host of the API gateway (optional). If not set, it will be automatically generated
         :param canary: The canary percents for the API gateway of type list[int]; for instance: [20,80]
         """
-        self.functions = None
-        self._validate(
-            project=project,
-            functions=functions,
-            name=name,
-            canary=canary,
-        )
-        self.project = project
-        self.name = name
+        self.description = description
         self.host = host
-
         self.path = path
-        self.description = description
-        self.canary = canary
         self.authentication = authentication
+        self.functions = functions
+        self.canary = canary
+        self.project = project
+
+        self.validate(project=project, functions=functions, canary=canary)
+
+    def validate(
+        self,
+        project: str,
+        functions: Union[
+            list[str],
+            Union[
+                list[
+                    Union[
+                        RemoteRuntime,
+                        ServingRuntime,
+                    ]
+                ],
+                RemoteRuntime,
+                ServingRuntime,
+            ],
+        ],
+        canary: Optional[list[int]] = None,
+    ):
+        self.functions = self._validate_functions(project=project, functions=functions)
+
+        # validating canary
+        if canary:
+            self.canary = self._validate_canary(canary)
+
+    def _validate_canary(self, canary: list[int]):
+        if len(self.functions) != len(canary):
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Function and canary lists lengths do not match"
+            )
+        for canary_percent in canary:
+            if canary_percent < 0 or canary_percent > 100:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "The percentage value must be in the range from 0 to 100"
+                )
+        if sum(canary) != 100:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "The sum of canary function percents should be equal to 100"
+            )
+        return canary
+
+    @staticmethod
+    def _validate_functions(
+        project: str,
+        functions: Union[
+            list[str],
+            Union[
+                list[
+                    Union[
+                        RemoteRuntime,
+                        ServingRuntime,
+                    ]
+                ],
+                Union[RemoteRuntime, ServingRuntime],
+            ],
+        ],
+    ):
+        if not isinstance(functions, list):
+            functions = [functions]
+
+        # validating functions
+        if not 1 <= len(functions) <= 2:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Gateway can be created from one or two functions, "
+                f"the number of functions passed is {len(functions)}"
+            )
+
+        function_names = []
+        for func in functions:
+            if isinstance(func, str):
+                function_names.append(func)
+                continue
+
+            function_name = (
+                func.metadata.name if hasattr(func, "metadata") else func.name
+            )
+            if func.kind not in mlrun.runtimes.RuntimeKinds.nuclio_runtimes():
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    f"Input function {function_name} is not a Nuclio function"
+                )
+            if func.metadata.project != project:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    f"input function {function_name} "
+                    f"does not belong to this project"
+                )
+            nuclio_name = get_fullname(function_name, project, func.metadata.tag)
+            function_names.append(nuclio_name)
+        return function_names
+
+
+class APIGateway(ModelObj):
+    _dict_fields = [
+        "metadata",
+        "spec",
+        "state",
+    ]
+
+    @min_nuclio_versions("1.13.1")
+    def __init__(
+        self,
+        metadata: APIGatewayMetadata,
+        spec: APIGatewaySpec,
+    ):
+        """
+        Initialize the APIGateway instance.
+
+        :param metadata: (APIGatewayMetadata) The metadata of the API gateway.
+        :param spec: (APIGatewaySpec) The spec of the API gateway.
+        """
+        self.metadata = metadata
+        self.spec = spec
         self.state = ""
 
+    @property
+    def metadata(self) -> APIGatewayMetadata:
+        return self._metadata
+
+    @metadata.setter
+    def metadata(self, metadata):
+        self._metadata = self._verify_dict(metadata, "metadata", APIGatewayMetadata)
+
+    @property
+    def spec(self) -> APIGatewaySpec:
+        return self._spec
+
+    @spec.setter
+    def spec(self, spec):
+        self._spec = self._verify_dict(spec, "spec", APIGatewaySpec)
+
     def invoke(
         self,
         method="POST",
@@ -181,7 +339,7 @@ class APIGateway:
             )
 
         if (
-            self.authentication.authentication_mode
+            self.spec.authentication.authentication_mode
             == NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH
             and not auth
         ):
@@ -227,15 +385,17 @@ class APIGateway:
         """
         Synchronize the API gateway from the server.
         """
-        synced_gateway = mlrun.get_run_db().get_api_gateway(self.name, self.project)
+        synced_gateway = mlrun.get_run_db().get_api_gateway(
+            self.metadata.name, self.spec.project
+        )
         synced_gateway = self.from_scheme(synced_gateway)
 
-        self.host = synced_gateway.host
-        self.path = synced_gateway.path
-        self.authentication = synced_gateway.authentication
-        self.functions = synced_gateway.functions
-        self.canary = synced_gateway.canary
-        self.description = synced_gateway.description
+        self.spec.host = synced_gateway.spec.host
+        self.spec.path = synced_gateway.spec.path
+        self.spec.authentication = synced_gateway.spec.authentication
+        self.spec.functions = synced_gateway.spec.functions
+        self.spec.canary = synced_gateway.spec.canary
+        self.spec.description = synced_gateway.spec.description
         self.state = synced_gateway.state
 
     def with_basic_auth(self, username: str, password: str):
@@ -245,7 +405,7 @@ class APIGateway:
         :param username: (str) The username for basic authentication.
         :param password: (str) The password for basic authentication.
         """
-        self.authentication = BasicAuth(username=username, password=password)
+        self.spec.authentication = BasicAuth(username=username, password=password)
 
     def with_canary(
         self,
@@ -276,8 +436,9 @@ class APIGateway:
                 f"Gateway with canary can be created only with two functions, "
                 f"the number of functions passed is {len(functions)}"
             )
-        self.functions = self._validate_functions(self.project, functions)
-        self.canary = self._validate_canary(canary)
+        self.spec.validate(
+            project=self.spec.project, functions=functions, canary=canary
+        )
 
     @classmethod
     def from_scheme(cls, api_gateway: mlrun.common.schemas.APIGateway):
@@ -288,54 +449,60 @@ class APIGateway:
             if api_gateway.status
             else mlrun.common.schemas.APIGatewayState.none
         )
-        api_gateway = cls(
-            project=project,
-            description=api_gateway.spec.description,
-            name=api_gateway.spec.name,
-            host=api_gateway.spec.host,
-            path=api_gateway.spec.path,
-            authentication=APIGatewayAuthenticator.from_scheme(api_gateway.spec),
-            functions=functions,
-            canary=canary,
+        new_api_gateway = cls(
+            metadata=APIGatewayMetadata(
+                name=api_gateway.spec.name,
+            ),
+            spec=APIGatewaySpec(
+                project=project,
+                description=api_gateway.spec.description,
+                host=api_gateway.spec.host,
+                path=api_gateway.spec.path,
+                authentication=APIGatewayAuthenticator.from_scheme(api_gateway.spec),
+                functions=functions,
+                canary=canary,
+            ),
         )
-        api_gateway.state = state
-        return api_gateway
+        new_api_gateway.state = state
+        return new_api_gateway
 
     def to_scheme(self) -> mlrun.common.schemas.APIGateway:
         upstreams = (
             [
                 mlrun.common.schemas.APIGatewayUpstream(
-                    nucliofunction={"name": self.functions[0]},
-                    percentage=self.canary[0],
+                    nucliofunction={"name": self.spec.functions[0]},
+                    percentage=self.spec.canary[0],
                 ),
                 mlrun.common.schemas.APIGatewayUpstream(
                     # do not set percent for the second function,
                     # so we can define which function to display as a primary one in UI
-                    nucliofunction={"name": self.functions[1]},
+                    nucliofunction={"name": self.spec.functions[1]},
                 ),
             ]
-            if self.canary
+            if self.spec.canary
             else [
                 mlrun.common.schemas.APIGatewayUpstream(
                     nucliofunction={"name": function_name},
                 )
-                for function_name in self.functions
+                for function_name in self.spec.functions
             ]
         )
         api_gateway = mlrun.common.schemas.APIGateway(
-            metadata=mlrun.common.schemas.APIGatewayMetadata(name=self.name, labels={}),
+            metadata=mlrun.common.schemas.APIGatewayMetadata(
+                name=self.metadata.name, labels={}
+            ),
             spec=mlrun.common.schemas.APIGatewaySpec(
-                name=self.name,
-                description=self.description,
-                host=self.host,
-                path=self.path,
+                name=self.metadata.name,
+                description=self.spec.description,
+                host=self.spec.host,
+                path=self.spec.path,
                 authenticationMode=mlrun.common.schemas.APIGatewayAuthenticationMode.from_str(
-                    self.authentication.authentication_mode
+                    self.spec.authentication.authentication_mode
                 ),
                 upstreams=upstreams,
             ),
         )
-        api_gateway.spec.authentication = self.authentication.to_scheme()
+        api_gateway.spec.authentication = self.spec.authentication.to_scheme()
         return api_gateway
 
     @property
@@ -347,103 +514,10 @@ class APIGateway:
 
         :return: (str) The invoke URL.
         """
-        host = self.host
-        if not self.host.startswith("http"):
-            host = f"https://{self.host}"
-        return urljoin(host, self.path)
-
-    def _validate(
-        self,
-        name: str,
-        project: str,
-        functions: Union[
-            list[str],
-            Union[
-                list[
-                    Union[
-                        RemoteRuntime,
-                        ServingRuntime,
-                    ]
-                ],
-                Union[RemoteRuntime, ServingRuntime],
-            ],
-        ],
-        canary: Optional[list[int]] = None,
-    ):
-        if not name:
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "API Gateway name cannot be empty"
-            )
-
-        self.functions = self._validate_functions(project=project, functions=functions)
-
-        # validating canary
-        if canary:
-            self._validate_canary(canary)
-
-    def _validate_canary(self, canary: list[int]):
-        if len(self.functions) != len(canary):
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "Function and canary lists lengths do not match"
-            )
-        for canary_percent in canary:
-            if canary_percent < 0 or canary_percent > 100:
-                raise mlrun.errors.MLRunInvalidArgumentError(
-                    "The percentage value must be in the range from 0 to 100"
-                )
-        if sum(canary) != 100:
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "The sum of canary function percents should be equal to 100"
-            )
-        return canary
-
-    @staticmethod
-    def _validate_functions(
-        project: str,
-        functions: Union[
-            list[str],
-            Union[
-                list[
-                    Union[
-                        RemoteRuntime,
-                        ServingRuntime,
-                    ]
-                ],
-                Union[RemoteRuntime, ServingRuntime],
-            ],
-        ],
-    ):
-        if not isinstance(functions, list):
-            functions = [functions]
-
-        # validating functions
-        if not 1 <= len(functions) <= 2:
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                f"Gateway can be created from one or two functions, "
-                f"the number of functions passed is {len(functions)}"
-            )
-
-        function_names = []
-        for func in functions:
-            if isinstance(func, str):
-                function_names.append(func)
-                continue
-
-            function_name = (
-                func.metadata.name if hasattr(func, "metadata") else func.name
-            )
-            if func.kind not in mlrun.runtimes.RuntimeKinds.nuclio_runtimes():
-                raise mlrun.errors.MLRunInvalidArgumentError(
-                    f"Input function {function_name} is not a Nuclio function"
-                )
-            if func.metadata.project != project:
-                raise mlrun.errors.MLRunInvalidArgumentError(
-                    f"input function {function_name} "
-                    f"does not belong to this project"
-                )
-            nuclio_name = get_fullname(function_name, project, func.metadata.tag)
-            function_names.append(nuclio_name)
-        return function_names
+        host = self.spec.host
+        if not self.spec.host.startswith("http"):
+            host = f"https://{self.spec.host}"
+        return urljoin(host, self.spec.path)
 
     @staticmethod
     def _generate_basic_auth(username: str, password: str):
@@ -475,3 +549,51 @@ class APIGateway:
         else:
             # Nuclio only supports 1 or 2 upstream functions
             return None, None
+
+    @property
+    def name(self):
+        return self.metadata.name
+
+    @name.setter
+    def name(self, value):
+        self.metadata.name = value
+
+    @property
+    def project(self):
+        return self.spec.project
+
+    @project.setter
+    def project(self, value):
+        self.spec.project = value
+
+    @property
+    def description(self):
+        return self.spec.description
+
+    @description.setter
+    def description(self, value):
+        self.spec.description = value
+
+    @property
+    def host(self):
+        return self.spec.host
+
+    @host.setter
+    def host(self, value):
+        self.spec.host = value
+
+    @property
+    def path(self):
+        return self.spec.path
+
+    @path.setter
+    def path(self, value):
+        self.spec.path = value
+
+    @property
+    def authentication(self):
+        return self.spec.authentication
+
+    @authentication.setter
+    def authentication(self, value):
+        self.spec.authentication = value

mlrun/runtimes/pod.py

@@ -1086,12 +1086,12 @@ class KubeResource(BaseRuntime):
 
     def _set_env(self, name, value=None, value_from=None):
         new_var = k8s_client.V1EnvVar(name=name, value=value, value_from=value_from)
-        i = 0
-        for v in self.spec.env:
-            if get_item_name(v) == name:
-                self.spec.env[i] = new_var
+
+        # ensure we don't have duplicate env vars with the same name
+        for env_index, value_item in enumerate(self.spec.env):
+            if get_item_name(value_item) == name:
+                self.spec.env[env_index] = new_var
                 return self
-            i += 1
         self.spec.env.append(new_var)
         return self
 

mlrun/serving/states.py

@@ -19,7 +19,7 @@ import pathlib
 import traceback
 from copy import copy, deepcopy
 from inspect import getfullargspec, signature
-from typing import Union
+from typing import Any, Union
 
 import mlrun
 
@@ -327,7 +327,7 @@ class BaseStep(ModelObj):
             parent = self._parent
         else:
             raise GraphError(
-                f"step {self.name} parent is not set or its not part of a graph"
+                f"step {self.name} parent is not set or it's not part of a graph"
             )
 
         name, step = params_to_step(
@@ -349,6 +349,36 @@ class BaseStep(ModelObj):
         parent._last_added = step
         return step
 
+    def set_flow(
+        self,
+        steps: list[Union[str, StepToDict, dict[str, Any]]],
+        force: bool = False,
+    ):
+        """set list of steps as downstream from this step, in the order specified. This will overwrite any existing
+        downstream steps.
+
+        :param steps: list of steps to follow this one
+        :param force: whether to overwrite existing downstream steps. If False, this method will fail if any downstream
+        steps have already been defined. Defaults to False.
+        :return: the last step added to the flow
+
+        example:
+            The below code sets the downstream nodes of step1 by using a list of steps (provided to `set_flow()`) and a
+            single step (provided to `to()`), resulting in the graph (step1 -> step2 -> step3 -> step4).
+            Notice that using `force=True` is required in case step1 already had downstream nodes (e.g. if the existing
+            graph is step1 -> step2_old) and that following the execution of this code the existing downstream steps
+            are removed. If the intention is to split the graph (and not to overwrite), please use `to()`.
+
+            step1.set_flow(
+                [
+                    dict(name="step2", handler="step2_handler"),
+                    dict(name="step3", class_name="Step3Class"),
+                ],
+                force=True,
+            ).to(dict(name="step4", class_name="Step4Class"))
+        """
+        raise NotImplementedError("set_flow() can only be called on a FlowStep")
+
 
 class TaskStep(BaseStep):
     """task execution step, runs a class or handler"""
@@ -1258,6 +1288,27 @@ class FlowStep(BaseStep):
                 )
             self[step_name].after_step(name)
 
+    def set_flow(
+        self,
+        steps: list[Union[str, StepToDict, dict[str, Any]]],
+        force: bool = False,
+    ):
+        if not force and self.steps:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "set_flow() called on a step that already has downstream steps. "
+                "If you want to overwrite existing steps, set force=True."
+            )
+
+        self.steps = None
+        step = self
+        for next_step in steps:
+            if isinstance(next_step, dict):
+                step = step.to(**next_step)
+            else:
+                step = step.to(next_step)
+
+        return step
+
 
 class RootFlowStep(FlowStep):
     """root flow step"""