mlrun 1.7.0rc14__py3-none-any.whl → 1.7.0rc16__py3-none-any.whl

mlrun/runtimes/funcdoc.py

@@ -16,8 +16,6 @@ import ast
 import inspect
 import re
 
-from deprecated import deprecated
-
 from mlrun.model import FunctionEntrypoint
 
 
@@ -73,32 +71,6 @@ def func_dict(
     }
 
 
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.5.0",
-    reason="'func_info' is deprecated and will be removed in 1.7.0, use 'ast_func_info' instead",
-    category=FutureWarning,
-)
-def func_info(fn) -> dict:
-    sig = inspect.signature(fn)
-    doc = inspect.getdoc(fn) or ""
-
-    out = func_dict(
-        name=fn.__name__,
-        doc=doc,
-        params=[inspect_param(p) for p in sig.parameters.values()],
-        returns=param_dict(
-            type=type_name(sig.return_annotation, empty_is_none=True), default=None
-        ),
-        lineno=func_lineno(fn),
-    )
-
-    if not fn.__doc__ or not fn.__doc__.strip():
-        return out
-
-    return merge_doc(out, doc)
-
-
 def func_lineno(fn):
     try:
         return inspect.getsourcelines(fn)[1]

mlrun/runtimes/kubejob.py

@@ -14,11 +14,12 @@
 
 import warnings
 
+from mlrun_pipelines.common.ops import build_op
+
 import mlrun.common.schemas
 import mlrun.db
 import mlrun.errors
 
-from ..kfpops import build_op
 from ..model import RunObject
 from .pod import KubeResource
 

mlrun/runtimes/local.py

@@ -493,7 +493,7 @@ def exec_from_params(handler, runobj: RunObject, context: MLClientCtx, cwd=None)
                 logger.warning("Run was aborted", err=err_to_str(exc))
                 # Run was aborted, the state run state is updated by the abort job, no need to commit again
                 context.set_state(
-                    mlrun.runtimes.constants.RunStates.aborted, commit=False
+                    mlrun.common.runtimes.constants.RunStates.aborted, commit=False
                 )
                 commit = False
             except Exception as exc:

mlrun/runtimes/mpijob/__init__.py

@@ -21,28 +21,8 @@ from mlrun.config import config
 from .. import MPIJobCRDVersions
 from .abstract import AbstractMPIJobRuntime
 from .v1 import MpiRuntimeV1
-from .v1alpha1 import MpiRuntimeV1Alpha1
 
 
 def _resolve_mpijob_crd_version():
     # config is expected to get enriched from the API through the client-spec
     return config.mpijob_crd_version or MPIJobCRDVersions.default()
-
-
-class MpiRuntimeContainer(containers.DeclarativeContainer):
-    resolver = providers.Callable(
-        _resolve_mpijob_crd_version,
-    )
-
-    selector = providers.Selector(
-        resolver,
-        v1=providers.Object(MpiRuntimeV1),
-        v1alpha1=providers.Object(MpiRuntimeV1Alpha1),
-    )
-
-    # An empty selector to be overriden by the API
-    handler_selector = providers.Selector(
-        resolver,
-        v1=providers.Object(None),
-        v1alpha1=providers.Object(None),
-    )

mlrun/runtimes/mpijob/v1.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from mlrun.runtimes.constants import MPIJobCRDVersions, MPIJobV1CleanPodPolicies
+from mlrun.common.runtimes.constants import MPIJobCRDVersions, MPIJobV1CleanPodPolicies
 from mlrun.runtimes.mpijob.abstract import AbstractMPIJobRuntime, MPIResourceSpec
 
 

mlrun/runtimes/nuclio/api_gateway.py

@@ -17,10 +17,12 @@ from typing import Optional, Union
 from urllib.parse import urljoin
 
 import requests
+from nuclio.auth import AuthInfo as NuclioAuthInfo
 from requests.auth import HTTPBasicAuth
 
 import mlrun
 import mlrun.common.schemas
+from mlrun.platforms.iguazio import min_iguazio_versions
 
 from ...model import ModelObj
 from ..utils import logger
@@ -29,6 +31,7 @@ from .serving import ServingRuntime
 
 NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH = "basicAuth"
 NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_NONE = "none"
+NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_ACCESS_KEY = "accessKey"
 PROJECT_NAME_LABEL = "nuclio.io/project-name"
 
 
@@ -50,6 +53,11 @@ class APIGatewayAuthenticator(typing.Protocol):
                 )
             else:
                 return BasicAuth()
+        elif (
+            api_gateway_spec.authenticationMode
+            == NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_ACCESS_KEY
+        ):
+            return AccessKeyAuth()
         else:
             return NoneAuth()
 
@@ -93,6 +101,16 @@ class BasicAuth(APIGatewayAuthenticator):
         }
 
 
+class AccessKeyAuth(APIGatewayAuthenticator):
+    """
+    An API gateway authenticator with access key authentication.
+    """
+
+    @property
+    def authentication_mode(self) -> str:
+        return NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_ACCESS_KEY
+
+
 class APIGatewayMetadata(ModelObj):
     _dict_fields = ["name", "namespace", "labels", "annotations", "creation_timestamp"]
 
@@ -156,6 +174,7 @@ class APIGatewaySpec(ModelObj):
         path: str = "/",
         authentication: Optional[APIGatewayAuthenticator] = NoneAuth(),
         canary: Optional[list[int]] = None,
+        ports: Optional[list[int]] = None,
     ):
         """
         :param functions: The list of functions associated with the API gateway
@@ -169,7 +188,9 @@ class APIGatewaySpec(ModelObj):
         :param authentication: The authentication for the API gateway of type
                 :py:class:`~mlrun.runtimes.nuclio.api_gateway.BasicAuth`
         :param host:  The host of the API gateway (optional). If not set, it will be automatically generated
-        :param canary: The canary percents for the API gateway of type list[int]; for instance: [20,80]
+        :param canary: The canary percents for the API gateway of type list[int]; for instance: [20,80] (optional)
+        :param ports: The ports of the API gateway, as a list of integers that correspond to the functions in the
+            functions list. for instance: [8050] or [8050, 8081] (optional)
         """
         self.description = description
         self.host = host
@@ -178,8 +199,9 @@ class APIGatewaySpec(ModelObj):
         self.functions = functions
         self.canary = canary
         self.project = project
+        self.ports = ports
 
-        self.validate(project=project, functions=functions, canary=canary)
+        self.validate(project=project, functions=functions, canary=canary, ports=ports)
 
     def validate(
         self,
@@ -198,6 +220,7 @@ class APIGatewaySpec(ModelObj):
             ],
         ],
         canary: Optional[list[int]] = None,
+        ports: Optional[list[int]] = None,
     ):
         self.functions = self._validate_functions(project=project, functions=functions)
 
@@ -205,6 +228,10 @@ class APIGatewaySpec(ModelObj):
         if canary:
             self.canary = self._validate_canary(canary)
 
+        # validating ports
+        if ports:
+            self.ports = self._validate_ports(ports)
+
     def _validate_canary(self, canary: list[int]):
         if len(self.functions) != len(canary):
             raise mlrun.errors.MLRunInvalidArgumentError(
@@ -221,6 +248,14 @@ class APIGatewaySpec(ModelObj):
             )
         return canary
 
+    def _validate_ports(self, ports):
+        if len(self.functions) != len(ports):
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Function and port lists lengths do not match"
+            )
+
+        return ports
+
     @staticmethod
     def _validate_functions(
         project: str,
@@ -312,7 +347,7 @@ class APIGateway(ModelObj):
     def invoke(
         self,
         method="POST",
-        headers: dict = {},
+        headers: dict = None,
         auth: Optional[tuple[str, str]] = None,
         **kwargs,
     ):
@@ -341,17 +376,26 @@ class APIGateway(ModelObj):
         if (
             self.spec.authentication.authentication_mode
             == NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH
-            and not auth
         ):
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "API Gateway invocation requires authentication. Please pass credentials"
-            )
+            if not auth:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "API Gateway invocation requires authentication. Please pass credentials"
+                )
+            auth = HTTPBasicAuth(*auth)
+
+        if (
+            self.spec.authentication.authentication_mode
+            == NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_ACCESS_KEY
+        ):
+            # inject access key from env
+            auth = NuclioAuthInfo().from_envvar().to_requests_auth()
+
         return requests.request(
             method=method,
             url=self.invoke_url,
-            headers=headers,
+            headers=headers or {},
             **kwargs,
-            auth=HTTPBasicAuth(*auth) if auth else None,
+            auth=auth,
         )
 
     def wait_for_readiness(self, max_wait_time=90):
@@ -407,6 +451,13 @@ class APIGateway(ModelObj):
         """
         self.spec.authentication = BasicAuth(username=username, password=password)
 
+    @min_iguazio_versions("3.5.5")
+    def with_access_key_auth(self):
+        """
+        Set access key authentication for the API gateway.
+        """
+        self.spec.authentication = AccessKeyAuth()
+
     def with_canary(
         self,
         functions: Union[
@@ -440,6 +491,17 @@ class APIGateway(ModelObj):
             project=self.spec.project, functions=functions, canary=canary
         )
 
+    def with_ports(self, ports: list[int]):
+        """
+        Set ports for the API gateway
+
+        :param ports: The ports of the API gateway, as a list of integers that correspond to the functions in the
+            functions list. for instance: [8050] or [8050, 8081]
+        """
+        self.spec.validate(
+            project=self.spec.project, functions=self.spec.functions, ports=ports
+        )
+
     @classmethod
     def from_scheme(cls, api_gateway: mlrun.common.schemas.APIGateway):
         project = api_gateway.metadata.labels.get(PROJECT_NAME_LABEL)
@@ -487,6 +549,10 @@ class APIGateway(ModelObj):
                 for function_name in self.spec.functions
             ]
         )
+        if self.spec.ports:
+            for i, port in enumerate(self.spec.ports):
+                upstreams[i].port = port
+
         api_gateway = mlrun.common.schemas.APIGateway(
             metadata=mlrun.common.schemas.APIGatewayMetadata(
                 name=self.metadata.name, labels={}

mlrun/runtimes/nuclio/function.py

@@ -22,9 +22,11 @@ from time import sleep
 import nuclio
 import nuclio.utils
 import requests
-import semver
 from aiohttp.client import ClientSession
 from kubernetes import client
+from mlrun_pipelines.common.mounts import VolumeMount
+from mlrun_pipelines.common.ops import deploy_op
+from mlrun_pipelines.mounts import mount_v3io, v3io_cred
 from nuclio.deploy import find_dashboard_url, get_deploy_status
 from nuclio.triggers import V3IOStreamTrigger
 
@@ -36,15 +38,11 @@ import mlrun.utils.helpers
 from mlrun.common.schemas import AuthInfo
 from mlrun.config import config as mlconf
 from mlrun.errors import err_to_str
-from mlrun.kfpops import deploy_op
 from mlrun.lists import RunList
 from mlrun.model import RunObject
 from mlrun.platforms.iguazio import (
-    VolumeMount,
-    mount_v3io,
     parse_path,
     split_path,
-    v3io_cred,
 )
 from mlrun.runtimes.base import FunctionStatus, RunError
 from mlrun.runtimes.pod import KubeResource, KubeResourceSpec
@@ -56,33 +54,9 @@ def validate_nuclio_version_compatibility(*min_versions):
     """
     :param min_versions: Valid minimum version(s) required, assuming no 2 versions has equal major and minor.
     """
-    parsed_min_versions = [
-        semver.VersionInfo.parse(min_version) for min_version in min_versions
-    ]
-    try:
-        parsed_current_version = semver.VersionInfo.parse(mlconf.nuclio_version)
-    except ValueError:
-        # only log when version is set but invalid
-        if mlconf.nuclio_version:
-            logger.warning(
-                "Unable to parse nuclio version, assuming compatibility",
-                nuclio_version=mlconf.nuclio_version,
-                min_versions=min_versions,
-            )
-        return True
-
-    parsed_min_versions.sort(reverse=True)
-    for parsed_min_version in parsed_min_versions:
-        if (
-            parsed_current_version.major == parsed_min_version.major
-            and parsed_current_version.minor == parsed_min_version.minor
-            and parsed_current_version.patch < parsed_min_version.patch
-        ):
-            return False
-
-        if parsed_current_version >= parsed_min_version:
-            return True
-    return False
+    return mlrun.utils.helpers.validate_component_version_compatibility(
+        "nuclio", *min_versions
+    )
 
 
 def min_nuclio_versions(*versions):
@@ -778,7 +752,7 @@ class RemoteRuntime(KubeResource):
             runtime_env["MLRUN_NAMESPACE"] = mlconf.namespace
         if self.metadata.credentials.access_key:
             runtime_env[
-                mlrun.runtimes.constants.FunctionEnvironmentVariables.auth_session
+                mlrun.common.runtimes.constants.FunctionEnvironmentVariables.auth_session
             ] = self.metadata.credentials.access_key
         return runtime_env
 
@@ -994,11 +968,11 @@ class RemoteRuntime(KubeResource):
         ports = mlrun.utils.helpers.as_list(ports)
         sidecar["ports"] = [
             {
-                "name": "http",
+                "name": f"{name}-{i}",
                 "containerPort": port,
                 "protocol": "TCP",
             }
-            for port in ports
+            for i, port in enumerate(ports)
         ]
 
         if command:

mlrun/runtimes/pod.py

@@ -20,8 +20,9 @@ import typing
 from enum import Enum
 
 import dotenv
-import kfp.dsl
 import kubernetes.client as k8s_client
+import mlrun_pipelines.mounts
+from mlrun_pipelines.mixins import KfpAdapterMixin
 
 import mlrun.errors
 import mlrun.utils.regex
@@ -41,7 +42,6 @@ from ..k8s_utils import (
 from ..utils import logger, update_in
 from .base import BaseRuntime, FunctionSpec, spec_fields
 from .utils import (
-    apply_kfp,
     get_gpu_from_resource_requirement,
     get_item_name,
     set_named_item,
@@ -935,12 +935,12 @@ class AutoMountType(str, Enum):
     @classmethod
     def all_mount_modifiers(cls):
         return [
-            mlrun.v3io_cred.__name__,
-            mlrun.mount_v3io.__name__,
-            mlrun.platforms.other.mount_pvc.__name__,
-            mlrun.auto_mount.__name__,
-            mlrun.platforms.mount_s3.__name__,
-            mlrun.platforms.set_env_variables.__name__,
+            mlrun_pipelines.mounts.v3io_cred.__name__,
+            mlrun_pipelines.mounts.mount_v3io.__name__,
+            mlrun_pipelines.mounts.mount_pvc.__name__,
+            mlrun_pipelines.mounts.auto_mount.__name__,
+            mlrun_pipelines.mounts.mount_s3.__name__,
+            mlrun_pipelines.mounts.set_env_variables.__name__,
         ]
 
     @classmethod
@@ -957,27 +957,27 @@ class AutoMountType(str, Enum):
     def _get_auto_modifier():
         # If we're running on Iguazio - use v3io_cred
         if mlconf.igz_version != "":
-            return mlrun.v3io_cred
+            return mlrun_pipelines.mounts.v3io_cred
         # Else, either pvc mount if it's configured or do nothing otherwise
         pvc_configured = (
             "MLRUN_PVC_MOUNT" in os.environ
             or "pvc_name" in mlconf.get_storage_auto_mount_params()
         )
-        return mlrun.platforms.other.mount_pvc if pvc_configured else None
+        return mlrun_pipelines.mounts.mount_pvc if pvc_configured else None
 
     def get_modifier(self):
         return {
             AutoMountType.none: None,
-            AutoMountType.v3io_credentials: mlrun.v3io_cred,
-            AutoMountType.v3io_fuse: mlrun.mount_v3io,
-            AutoMountType.pvc: mlrun.platforms.other.mount_pvc,
+            AutoMountType.v3io_credentials: mlrun_pipelines.mounts.v3io_cred,
+            AutoMountType.v3io_fuse: mlrun_pipelines.mounts.mount_v3io,
+            AutoMountType.pvc: mlrun_pipelines.mounts.mount_pvc,
             AutoMountType.auto: self._get_auto_modifier(),
-            AutoMountType.s3: mlrun.platforms.mount_s3,
-            AutoMountType.env: mlrun.platforms.set_env_variables,
+            AutoMountType.s3: mlrun_pipelines.mounts.mount_s3,
+            AutoMountType.env: mlrun_pipelines.mounts.set_env_variables,
         }[self]
 
 
-class KubeResource(BaseRuntime):
+class KubeResource(BaseRuntime, KfpAdapterMixin):
     """
     A parent class for runtimes that generate k8s resources when executing.
     """
@@ -997,26 +997,6 @@ class KubeResource(BaseRuntime):
     def spec(self, spec):
         self._spec = self._verify_dict(spec, "spec", KubeResourceSpec)
 
-    def apply(self, modify):
-        """
-        Apply a modifier to the runtime which is used to change the runtimes k8s object's spec.
-        Modifiers can be either KFP modifiers or MLRun modifiers (which are compatible with KFP). All modifiers accept
-        a `kfp.dsl.ContainerOp` object, apply some changes on its spec and return it so modifiers can be chained
-        one after the other.
-
-        :param modify: a modifier runnable object
-        :return: the runtime (self) after the modifications
-        """
-
-        # Kubeflow pipeline have a hook to add the component to the DAG on ContainerOp init
-        # we remove the hook to suppress kubeflow op registration and return it after the apply()
-        old_op_handler = kfp.dsl._container_op._register_op_handler
-        kfp.dsl._container_op._register_op_handler = lambda x: self.metadata.name
-        cop = kfp.dsl.ContainerOp("name", "image")
-        kfp.dsl._container_op._register_op_handler = old_op_handler
-
-        return apply_kfp(modify, cop, self)
-
     def set_env_from_secret(self, name, secret=None, secret_key=None):
         """set pod environment var from secret"""
         secret_key = secret_key or name

mlrun/runtimes/remotesparkjob.py

@@ -15,11 +15,11 @@ import re
 from subprocess import run
 
 import kubernetes.client
+from mlrun_pipelines.mounts import mount_v3io, mount_v3iod
 
 import mlrun.errors
 from mlrun.config import config
 
-from ..platforms.iguazio import mount_v3io, mount_v3iod
 from .kubejob import KubejobRuntime
 from .pod import KubeResourceSpec
 

mlrun/runtimes/sparkjob/spark3job.py

@@ -14,6 +14,7 @@
 import typing
 
 import kubernetes.client
+from mlrun_pipelines.mounts import mount_v3io, mount_v3iod
 
 import mlrun.common.schemas.function
 import mlrun.errors
@@ -22,7 +23,6 @@ from mlrun.config import config
 
 from ...execution import MLClientCtx
 from ...model import RunObject
-from ...platforms.iguazio import mount_v3io, mount_v3iod
 from ...utils import update_in, verify_field_regex
 from ..kubejob import KubejobRuntime
 from ..pod import KubeResourceSpec

mlrun/runtimes/utils.py

@@ -20,17 +20,16 @@ from io import StringIO
 from sys import stderr
 
 import pandas as pd
-from kubernetes import client
 
 import mlrun
 import mlrun.common.constants
 import mlrun.common.schemas
 import mlrun.utils.regex
 from mlrun.artifacts import TableArtifact
+from mlrun.common.runtimes.constants import RunLabels
 from mlrun.config import config
 from mlrun.errors import err_to_str
 from mlrun.frameworks.parallel_coordinates import gen_pcp_plot
-from mlrun.runtimes.constants import RunLabels
 from mlrun.runtimes.generators import selector
 from mlrun.utils import get_in, helpers, logger, verify_field_regex
 
@@ -280,43 +279,6 @@ def get_item_name(item, attr="name"):
         return getattr(item, attr, None)
 
 
-def apply_kfp(modify, cop, runtime):
-    modify(cop)
-
-    # Have to do it here to avoid circular dependencies
-    from .pod import AutoMountType
-
-    if AutoMountType.is_auto_modifier(modify):
-        runtime.spec.disable_auto_mount = True
-
-    api = client.ApiClient()
-    for k, v in cop.pod_labels.items():
-        runtime.metadata.labels[k] = v
-    for k, v in cop.pod_annotations.items():
-        runtime.metadata.annotations[k] = v
-    if cop.container.env:
-        env_names = [
-            e.name if hasattr(e, "name") else e["name"] for e in runtime.spec.env
-        ]
-        for e in api.sanitize_for_serialization(cop.container.env):
-            name = e["name"]
-            if name in env_names:
-                runtime.spec.env[env_names.index(name)] = e
-            else:
-                runtime.spec.env.append(e)
-                env_names.append(name)
-        cop.container.env.clear()
-
-    if cop.volumes and cop.container.volume_mounts:
-        vols = api.sanitize_for_serialization(cop.volumes)
-        mounts = api.sanitize_for_serialization(cop.container.volume_mounts)
-        runtime.spec.update_vols_and_mounts(vols, mounts)
-        cop.volumes.clear()
-        cop.container.volume_mounts.clear()
-
-    return runtime
-
-
 def verify_limits(
     resources_field_name,
     mem=None,