mlrun 1.7.0rc7__py3-none-any.whl → 1.7.0rc11__py3-none-any.whl

mlrun/run.py

@@ -114,16 +114,18 @@ def function_to_module(code="", workdir=None, secrets=None, silent=False):
 
     example::
 
-        mod = mlrun.function_to_module('./examples/training.py')
-        task = mlrun.new_task(inputs={'infile.txt': '../examples/infile.txt'})
-        context = mlrun.get_or_create_ctx('myfunc', spec=task)
-        mod.my_job(context, p1=1, p2='x')
+        mod = mlrun.function_to_module("./examples/training.py")
+        task = mlrun.new_task(inputs={"infile.txt": "../examples/infile.txt"})
+        context = mlrun.get_or_create_ctx("myfunc", spec=task)
+        mod.my_job(context, p1=1, p2="x")
         print(context.to_yaml())
 
-        fn = mlrun.import_function('hub://open-archive')
+        fn = mlrun.import_function("hub://open-archive")
         mod = mlrun.function_to_module(fn)
-        data = mlrun.run.get_dataitem("https://fpsignals-public.s3.amazonaws.com/catsndogs.tar.gz")
-        context = mlrun.get_or_create_ctx('myfunc')
+        data = mlrun.run.get_dataitem(
+            "https://fpsignals-public.s3.amazonaws.com/catsndogs.tar.gz"
+        )
+        context = mlrun.get_or_create_ctx("myfunc")
         mod.open_archive(context, archive_url=data)
         print(context.to_yaml())
 
@@ -256,29 +258,31 @@ def get_or_create_ctx(
     Examples::
 
         # load MLRUN runtime context (will be set by the runtime framework e.g. KubeFlow)
-        context = get_or_create_ctx('train')
+        context = get_or_create_ctx("train")
 
         # get parameters from the runtime context (or use defaults)
-        p1 = context.get_param('p1', 1)
-        p2 = context.get_param('p2', 'a-string')
+        p1 = context.get_param("p1", 1)
+        p2 = context.get_param("p2", "a-string")
 
         # access input metadata, values, files, and secrets (passwords)
-        print(f'Run: {context.name} (uid={context.uid})')
-        print(f'Params: p1={p1}, p2={p2}')
+        print(f"Run: {context.name} (uid={context.uid})")
+        print(f"Params: p1={p1}, p2={p2}")
         print(f'accesskey = {context.get_secret("ACCESS_KEY")}')
-        input_str = context.get_input('infile.txt').get()
-        print(f'file: {input_str}')
+        input_str = context.get_input("infile.txt").get()
+        print(f"file: {input_str}")
 
         # RUN some useful code e.g. ML training, data prep, etc.
 
         # log scalar result values (job result metrics)
-        context.log_result('accuracy', p1 * 2)
-        context.log_result('loss', p1 * 3)
-        context.set_label('framework', 'sklearn')
+        context.log_result("accuracy", p1 * 2)
+        context.log_result("loss", p1 * 3)
+        context.set_label("framework", "sklearn")
 
         # log various types of artifacts (file, web page, table), will be versioned and visible in the UI
-        context.log_artifact('model.txt', body=b'abc is 123', labels={'framework': 'xgboost'})
-        context.log_artifact('results.html', body=b'<b> Some HTML <b>', viewer='web-app')
+        context.log_artifact(
+            "model.txt", body=b"abc is 123", labels={"framework": "xgboost"}
+        )
+        context.log_artifact("results.html", body=b"<b> Some HTML <b>", viewer="web-app")
 
     """
 
@@ -348,7 +352,9 @@ def import_function(url="", secrets=None, db="", project=None, new_name=None):
 
         function = mlrun.import_function("hub://auto-trainer")
         function = mlrun.import_function("./func.yaml")
-        function = mlrun.import_function("https://raw.githubusercontent.com/org/repo/func.yaml")
+        function = mlrun.import_function(
+            "https://raw.githubusercontent.com/org/repo/func.yaml"
+        )
 
     :param url: path/url to Function Hub, db or function YAML file
     :param secrets: optional, credentials dict for DB or URL (s3, v3io, ...)
@@ -389,6 +395,8 @@ def import_function_to_dict(url, secrets=None):
     code = get_in(runtime, "spec.build.functionSourceCode")
     update_in(runtime, "metadata.build.code_origin", url)
     cmd = code_file = get_in(runtime, "spec.command", "")
+    # use kind = "job" by default if not specified
+    runtime.setdefault("kind", "job")
     if " " in cmd:
         code_file = cmd[: cmd.find(" ")]
     if runtime["kind"] in ["", "local"]:
@@ -445,12 +453,18 @@ def new_function(
     Example::
 
            # define a container based function (the `training.py` must exist in the container workdir)
-           f = new_function(command='training.py -x {x}', image='myrepo/image:latest', kind='job')
+           f = new_function(
+               command="training.py -x {x}", image="myrepo/image:latest", kind="job"
+           )
            f.run(params={"x": 5})
 
            # define a container based function which reads its source from a git archive
-           f = new_function(command='training.py -x {x}', image='myrepo/image:latest', kind='job',
-                            source='git://github.com/mlrun/something.git')
+           f = new_function(
+               command="training.py -x {x}",
+               image="myrepo/image:latest",
+               kind="job",
+               source="git://github.com/mlrun/something.git",
+           )
            f.run(params={"x": 5})
 
            # define a local handler function (execute a local function handler)
@@ -535,7 +549,7 @@ def new_function(
     if source:
         runner.spec.build.source = source
     if handler:
-        if kind in [RuntimeKinds.serving, RuntimeKinds.application]:
+        if kind in RuntimeKinds.handlerless_runtimes():
             raise MLRunInvalidArgumentError(
                 f"Handler is not supported for {kind} runtime"
             )
@@ -628,6 +642,8 @@ def code_to_function(
     - mpijob: run distributed Horovod jobs over the MPI job operator
     - spark: run distributed Spark job using Spark Kubernetes Operator
     - remote-spark: run distributed Spark job on remote Spark service
+    - databricks: run code on Databricks cluster (python scripts, Spark etc.)
+    - application: run a long living application (e.g. a web server, UI, etc.)
 
     Learn more about [Kinds of function (runtimes)](../concepts/functions-overview.html).
 
@@ -661,11 +677,15 @@ def code_to_function(
         import mlrun
 
         # create job function object from notebook code and add doc/metadata
-        fn = mlrun.code_to_function("file_utils", kind="job",
-                                    handler="open_archive", image="mlrun/mlrun",
-                                    description = "this function opens a zip archive into a local/mounted folder",
-                                    categories = ["fileutils"],
-                                    labels = {"author": "me"})
+        fn = mlrun.code_to_function(
+            "file_utils",
+            kind="job",
+            handler="open_archive",
+            image="mlrun/mlrun",
+            description="this function opens a zip archive into a local/mounted folder",
+            categories=["fileutils"],
+            labels={"author": "me"},
+        )
 
     example::
 
@@ -676,11 +696,15 @@ def code_to_function(
         Path("mover.py").touch()
 
         # create nuclio function object from python module call mover.py
-        fn = mlrun.code_to_function("nuclio-mover", kind="nuclio",
-                                    filename="mover.py", image="python:3.7",
-                                    description = "this function moves files from one system to another",
-                                    requirements = ["pandas"],
-                                    labels = {"author": "me"})
+        fn = mlrun.code_to_function(
+            "nuclio-mover",
+            kind="nuclio",
+            filename="mover.py",
+            image="python:3.7",
+            description="this function moves files from one system to another",
+            requirements=["pandas"],
+            labels={"author": "me"},
+        )
 
     """
     filebase, _ = path.splitext(path.basename(filename))
@@ -1094,13 +1118,25 @@ def wait_for_runs_completion(
     example::
 
         # run two training functions in parallel and wait for the results
-        inputs = {'dataset': cleaned_data}
-        run1 = train.run(name='train_lr', inputs=inputs, watch=False,
-                         params={'model_pkg_class': 'sklearn.linear_model.LogisticRegression',
-                                 'label_column': 'label'})
-        run2 = train.run(name='train_lr', inputs=inputs, watch=False,
-                         params={'model_pkg_class': 'sklearn.ensemble.RandomForestClassifier',
-                                 'label_column': 'label'})
+        inputs = {"dataset": cleaned_data}
+        run1 = train.run(
+            name="train_lr",
+            inputs=inputs,
+            watch=False,
+            params={
+                "model_pkg_class": "sklearn.linear_model.LogisticRegression",
+                "label_column": "label",
+            },
+        )
+        run2 = train.run(
+            name="train_lr",
+            inputs=inputs,
+            watch=False,
+            params={
+                "model_pkg_class": "sklearn.ensemble.RandomForestClassifier",
+                "label_column": "label",
+            },
+        )
         completed = wait_for_runs_completion([run1, run2])
 
     :param runs:    list of run objects (the returned values of function.run())

mlrun/runtimes/__init__.py

@@ -154,6 +154,22 @@ class RuntimeKinds:
             RuntimeKinds.application,
         ]
 
+    @staticmethod
+    def pure_nuclio_deployed_runtimes():
+        return [
+            RuntimeKinds.remote,
+            RuntimeKinds.nuclio,
+            RuntimeKinds.serving,
+        ]
+
+    @staticmethod
+    def handlerless_runtimes():
+        return [
+            RuntimeKinds.serving,
+            # Application runtime handler is internal reverse proxy
+            RuntimeKinds.application,
+        ]
+
     @staticmethod
     def local_runtimes():
         return [

mlrun/runtimes/base.py

@@ -23,6 +23,7 @@ from typing import Callable, Optional, Union
 import requests.exceptions
 from nuclio.build import mlrun_footer
 
+import mlrun.common.constants
 import mlrun.common.schemas
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.db
@@ -634,7 +635,9 @@ class BaseRuntime(ModelObj):
         image = image or self.spec.image or ""
 
         image = enrich_image_url(image, client_version, client_python_version)
-        if not image.startswith("."):
+        if not image.startswith(
+            mlrun.common.constants.IMAGE_NAME_ENRICH_REGISTRY_PREFIX
+        ):
             return image
         registry, repository = get_parsed_docker_registry()
         if registry:

mlrun/runtimes/kubejob.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import time
 import warnings
 
 import mlrun.common.schemas
@@ -21,7 +20,6 @@ import mlrun.errors
 
 from ..kfpops import build_op
 from ..model import RunObject
-from ..utils import get_in, logger
 from .pod import KubeResource
 
 
@@ -65,29 +63,13 @@ class KubejobRuntime(KubeResource):
         :param pull_at_runtime: load the archive into the container at job runtime vs on build/deploy
         :param target_dir:      target dir on runtime pod or repo clone / archive extraction
         """
-        mlrun.utils.helpers.validate_builder_source(source, pull_at_runtime, workdir)
-
-        self.spec.build.source = source
-        if handler:
-            self.spec.default_handler = handler
-        if workdir:
-            self.spec.workdir = workdir
-        if target_dir:
-            self.spec.build.source_code_target_dir = target_dir
-
-        self.spec.build.load_source_on_run = pull_at_runtime
-        if (
-            self.spec.build.base_image
-            and not self.spec.build.commands
-            and pull_at_runtime
-            and not self.spec.image
-        ):
-            # if we load source from repo and don't need a full build use the base_image as the image
-            self.spec.image = self.spec.build.base_image
-        elif not pull_at_runtime:
-            # clear the image so build will not be skipped
-            self.spec.build.base_image = self.spec.build.base_image or self.spec.image
-            self.spec.image = ""
+        self._configure_mlrun_build_with_source(
+            source=source,
+            workdir=workdir,
+            handler=handler,
+            pull_at_runtime=pull_at_runtime,
+            target_dir=target_dir,
+        )
 
     def build_config(
         self,
@@ -169,116 +151,39 @@ class KubejobRuntime(KubeResource):
         show_on_failure: bool = False,
         force_build: bool = False,
     ) -> bool:
-        """deploy function, build container with dependencies
+        """Deploy function, build container with dependencies
 
-        :param watch:                   wait for the deploy to complete (and print build logs)
-        :param with_mlrun:              add the current mlrun package to the container build
-        :param skip_deployed:           skip the build if we already have an image for the function
-        :param is_kfp:                  deploy as part of a kfp pipeline
-        :param mlrun_version_specifier: which mlrun package version to include (if not current)
+        :param watch:                   Wait for the deploy to complete (and print build logs)
+        :param with_mlrun:              Add the current mlrun package to the container build
+        :param skip_deployed:           Skip the build if we already have an image for the function
+        :param is_kfp:                  Deploy as part of a kfp pipeline
+        :param mlrun_version_specifier: Which mlrun package version to include (if not current)
         :param builder_env:             Kaniko builder pod env vars dict (for config/credentials)
                                         e.g. builder_env={"GIT_TOKEN": token}
-        :param show_on_failure:         show logs only in case of build failure
-        :param force_build:             set True for force building the image, even when no changes were made
+        :param show_on_failure:         Show logs only in case of build failure
+        :param force_build:             Set True for force building the image, even when no changes were made
 
         :return: True if the function is ready (deployed)
         """
 
         build = self.spec.build
+        with_mlrun = self._resolve_build_with_mlrun(with_mlrun)
 
-        if with_mlrun is None:
-            if build.with_mlrun is not None:
-                with_mlrun = build.with_mlrun
-            else:
-                with_mlrun = build.base_image and not (
-                    build.base_image.startswith("mlrun/")
-                    or "/mlrun/" in build.base_image
-                )
-
-        if (
-            not build.source
-            and not build.commands
-            and not build.requirements
-            and not build.extra
-            and with_mlrun
-        ):
-            logger.info(
-                "Running build to add mlrun package, set "
-                "with_mlrun=False to skip if its already in the image"
-            )
         self.status.state = ""
         if build.base_image:
             # clear the image so build will not be skipped
             self.spec.image = ""
 
-        # When we're in pipelines context we must watch otherwise the pipelines pod will exit before the operation
-        # is actually done. (when a pipelines pod exits, the pipeline step marked as done)
-        if is_kfp:
-            watch = True
-
-        ready = False
-        if self._is_remote_api():
-            db = self._get_db()
-            data = db.remote_builder(
-                self,
-                with_mlrun,
-                mlrun_version_specifier,
-                skip_deployed,
-                builder_env=builder_env,
-                force_build=force_build,
-            )
-            self.status = data["data"].get("status", None)
-            self.spec.image = get_in(data, "data.spec.image")
-            self.spec.build.base_image = self.spec.build.base_image or get_in(
-                data, "data.spec.build.base_image"
-            )
-            # Get the source target dir in case it was enriched due to loading source
-            self.spec.build.source_code_target_dir = get_in(
-                data, "data.spec.build.source_code_target_dir"
-            ) or get_in(data, "data.spec.clone_target_dir")
-            ready = data.get("ready", False)
-            if not ready:
-                logger.info(
-                    f"Started building image: {data.get('data', {}).get('spec', {}).get('build', {}).get('image')}"
-                )
-            if watch and not ready:
-                state = self._build_watch(watch, show_on_failure=show_on_failure)
-                ready = state == "ready"
-                self.status.state = state
-
-        if watch and not ready:
-            raise mlrun.errors.MLRunRuntimeError("Deploy failed")
-        return ready
-
-    def _build_watch(self, watch=True, logs=True, show_on_failure=False):
-        db = self._get_db()
-        offset = 0
-        try:
-            text, _ = db.get_builder_status(self, 0, logs=logs)
-        except mlrun.db.RunDBError:
-            raise ValueError("function or build process not found")
-
-        def print_log(text):
-            if text and (not show_on_failure or self.status.state == "error"):
-                print(text, end="")
-
-        print_log(text)
-        offset += len(text)
-        if watch:
-            while self.status.state in ["pending", "running"]:
-                time.sleep(2)
-                if show_on_failure:
-                    text = ""
-                    db.get_builder_status(self, 0, logs=False)
-                    if self.status.state == "error":
-                        # re-read the full log on failure
-                        text, _ = db.get_builder_status(self, offset, logs=logs)
-                else:
-                    text, _ = db.get_builder_status(self, offset, logs=logs)
-                print_log(text)
-                offset += len(text)
-
-        return self.status.state
+        return self._build_image(
+            builder_env=builder_env,
+            force_build=force_build,
+            mlrun_version_specifier=mlrun_version_specifier,
+            show_on_failure=show_on_failure,
+            skip_deployed=skip_deployed,
+            watch=watch,
+            is_kfp=is_kfp,
+            with_mlrun=with_mlrun,
+        )
 
     def deploy_step(
         self,

mlrun/runtimes/mpijob/abstract.py

@@ -223,14 +223,14 @@ class AbstractMPIJobRuntime(KubejobRuntime, abc.ABC):
         ```
         # Define the wanted MPI arguments
         mpi_args = []
-        mpi_args.append('-x')
-        mpi_args.append('NCCL_DEBUG=INFO')
-        mpi_args.append('-x')
-        mpi_args.append('NCCL_SOCKET_NTHREADS=2')
-        mpi_args.append('-x')
-        mpi_args.append('NCCL_NSOCKS_PERTHREAD=8')
-        mpi_args.append('-x')
-        mpi_args.append('NCCL_MIN_NCHANNELS=4')
+        mpi_args.append("-x")
+        mpi_args.append("NCCL_DEBUG=INFO")
+        mpi_args.append("-x")
+        mpi_args.append("NCCL_SOCKET_NTHREADS=2")
+        mpi_args.append("-x")
+        mpi_args.append("NCCL_NSOCKS_PERTHREAD=8")
+        mpi_args.append("-x")
+        mpi_args.append("NCCL_MIN_NCHANNELS=4")
 
         # Set the MPI arguments in the function
         fn.set_mpi_args(mpi_args)

mlrun/runtimes/nuclio/api_gateway.py

@@ -22,7 +22,8 @@ from requests.auth import HTTPBasicAuth
 import mlrun
 import mlrun.common.schemas
 
-from .function import RemoteRuntime, get_fullname
+from ..utils import logger
+from .function import RemoteRuntime, get_fullname, min_nuclio_versions
 from .serving import ServingRuntime
 
 NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH = "basicAuth"
@@ -85,13 +86,14 @@ class BasicAuth(APIGatewayAuthenticator):
         self,
     ) -> Optional[dict[str, Optional[mlrun.common.schemas.APIGatewayBasicAuth]]]:
         return {
-            "authentication": mlrun.common.schemas.APIGatewayBasicAuth(
+            "basicAuth": mlrun.common.schemas.APIGatewayBasicAuth(
                 username=self._username, password=self._password
             )
         }
 
 
 class APIGateway:
+    @min_nuclio_versions("1.13.1")
     def __init__(
         self,
         project,
@@ -147,6 +149,7 @@ class APIGateway:
         self.description = description
         self.canary = canary
         self.authentication = authentication
+        self.state = ""
 
     def invoke(
         self,
@@ -172,6 +175,11 @@ class APIGateway:
                 raise mlrun.errors.MLRunInvalidArgumentError(
                     "Invocation url is not set. Set up gateway's `invoke_url` attribute."
                 )
+        if not self.is_ready():
+            raise mlrun.errors.MLRunPreconditionFailedError(
+                f"API gateway is not ready. " f"Current state: {self.state}"
+            )
+
         if (
             self.authentication.authentication_mode
             == NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH
@@ -188,6 +196,33 @@ class APIGateway:
             auth=HTTPBasicAuth(*auth) if auth else None,
         )
 
+    def wait_for_readiness(self, max_wait_time=90):
+        """
+        Wait for the API gateway to become ready within the maximum wait time.
+
+        Parameters:
+            max_wait_time: int - Maximum time to wait in seconds (default is 90 seconds).
+
+        Returns:
+            bool: True if the entity becomes ready within the maximum wait time, False otherwise
+        """
+
+        def _ensure_ready():
+            if not self.is_ready():
+                raise AssertionError(
+                    f"Waiting for gateway readiness is taking more than {max_wait_time} seconds"
+                )
+
+        return mlrun.utils.helpers.retry_until_successful(
+            3, max_wait_time, logger, False, _ensure_ready
+        )
+
+    def is_ready(self):
+        if self.state is not mlrun.common.schemas.api_gateway.APIGatewayState.ready:
+            # try to sync the state
+            self.sync()
+        return self.state == mlrun.common.schemas.api_gateway.APIGatewayState.ready
+
     def sync(self):
         """
         Synchronize the API gateway from the server.
@@ -201,6 +236,7 @@ class APIGateway:
         self.functions = synced_gateway.functions
         self.canary = synced_gateway.canary
         self.description = synced_gateway.description
+        self.state = synced_gateway.state
 
     def with_basic_auth(self, username: str, password: str):
         """
@@ -247,7 +283,12 @@ class APIGateway:
     def from_scheme(cls, api_gateway: mlrun.common.schemas.APIGateway):
         project = api_gateway.metadata.labels.get(PROJECT_NAME_LABEL)
         functions, canary = cls._resolve_canary(api_gateway.spec.upstreams)
-        return cls(
+        state = (
+            api_gateway.status.state
+            if api_gateway.status
+            else mlrun.common.schemas.APIGatewayState.none
+        )
+        api_gateway = cls(
             project=project,
             description=api_gateway.spec.description,
             name=api_gateway.spec.name,
@@ -257,15 +298,21 @@ class APIGateway:
             functions=functions,
             canary=canary,
         )
+        api_gateway.state = state
+        return api_gateway
 
     def to_scheme(self) -> mlrun.common.schemas.APIGateway:
         upstreams = (
             [
                 mlrun.common.schemas.APIGatewayUpstream(
-                    nucliofunction={"name": function_name},
-                    percentage=percentage,
-                )
-                for function_name, percentage in zip(self.functions, self.canary)
+                    nucliofunction={"name": self.functions[0]},
+                    percentage=self.canary[0],
+                ),
+                mlrun.common.schemas.APIGatewayUpstream(
+                    # do not set percent for the second function,
+                    # so we can define which function to display as a primary one in UI
+                    nucliofunction={"name": self.functions[1]},
+                ),
             ]
             if self.canary
             else [
@@ -300,7 +347,10 @@ class APIGateway:
 
         :return: (str) The invoke URL.
         """
-        return urljoin(self.host, self.path)
+        host = self.host
+        if not self.host.startswith("http"):
+            host = f"https://{self.host}"
+        return urljoin(host, self.path)
 
     def _validate(
         self,