mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/launcher/local.py

@@ -0,0 +1,276 @@
+# Copyright 2023 MLRun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import pathlib
+from typing import Callable, Dict, List, Optional, Union
+
+import mlrun.common.schemas.schedule
+import mlrun.errors
+import mlrun.launcher.client
+import mlrun.run
+import mlrun.runtimes.generators
+import mlrun.utils.clones
+import mlrun.utils.notifications
+from mlrun.utils import logger
+
+
+class ClientLocalLauncher(mlrun.launcher.client.ClientBaseLauncher):
+    """
+    ClientLocalLauncher is a launcher that runs the job locally.
+    Either on the user's machine (_is_run_local is True) or on a remote machine (_is_run_local is False).
+    """
+
+    def __init__(self, local: bool):
+        """
+        Initialize a ClientLocalLauncher.
+        :param local:   True if the job runs on the user's local machine,
+                        False if it runs on a remote machine (e.g. a dedicated k8s pod).
+        """
+        super().__init__()
+        self._is_run_local = local
+
+    def launch(
+        self,
+        runtime: "mlrun.runtimes.BaseRuntime",
+        task: Optional[
+            Union["mlrun.run.RunTemplate", "mlrun.run.RunObject", dict]
+        ] = None,
+        handler: Optional[Union[str, Callable]] = None,
+        name: Optional[str] = "",
+        project: Optional[str] = "",
+        params: Optional[dict] = None,
+        inputs: Optional[Dict[str, str]] = None,
+        out_path: Optional[str] = "",
+        workdir: Optional[str] = "",
+        artifact_path: Optional[str] = "",
+        watch: Optional[bool] = True,
+        schedule: Optional[
+            Union[str, mlrun.common.schemas.schedule.ScheduleCronTrigger]
+        ] = None,
+        hyperparams: Dict[str, list] = None,
+        hyper_param_options: Optional[mlrun.model.HyperParamOptions] = None,
+        verbose: Optional[bool] = None,
+        scrape_metrics: Optional[bool] = None,
+        local_code_path: Optional[str] = None,
+        auto_build: Optional[bool] = None,
+        param_file_secrets: Optional[Dict[str, str]] = None,
+        notifications: Optional[List[mlrun.model.Notification]] = None,
+        returns: Optional[List[Union[str, Dict[str, str]]]] = None,
+    ) -> "mlrun.run.RunObject":
+
+        # do not allow local function to be scheduled
+        if self._is_run_local and schedule is not None:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "local and schedule cannot be used together"
+            )
+
+        self.enrich_runtime(runtime)
+        run = self._create_run_object(task)
+
+        if self._is_run_local:
+            runtime = self._create_local_function_for_execution(
+                runtime=runtime,
+                run=run,
+                local_code_path=local_code_path,
+                project=project,
+                name=name,
+                workdir=workdir,
+                handler=handler,
+            )
+
+        # sanity check
+        elif runtime._is_remote:
+            message = "Remote function cannot be executed locally"
+            logger.error(
+                message,
+                is_remote=runtime._is_remote,
+                local=self._is_run_local,
+                runtime=runtime.to_dict(),
+            )
+            raise mlrun.errors.MLRunRuntimeError(message)
+
+        run = self._enrich_run(
+            runtime=runtime,
+            run=run,
+            handler=handler,
+            project_name=project,
+            name=name,
+            params=params,
+            inputs=inputs,
+            returns=returns,
+            hyperparams=hyperparams,
+            hyper_param_options=hyper_param_options,
+            verbose=verbose,
+            scrape_metrics=scrape_metrics,
+            out_path=out_path,
+            artifact_path=artifact_path,
+            workdir=workdir,
+            notifications=notifications,
+        )
+        self._validate_runtime(runtime, run)
+        result = self.execute(
+            runtime=runtime,
+            run=run,
+        )
+
+        return result
+
+    def execute(
+        self,
+        runtime: "mlrun.runtimes.BaseRuntime",
+        run: Optional[Union["mlrun.run.RunTemplate", "mlrun.run.RunObject"]] = None,
+    ):
+
+        if "V3IO_USERNAME" in os.environ and "v3io_user" not in run.metadata.labels:
+            run.metadata.labels["v3io_user"] = os.environ.get("V3IO_USERNAME")
+
+        # store function object in db unless running from within a run pod
+        if not runtime.is_child:
+            logger.info(
+                "Storing function",
+                name=run.metadata.name,
+                uid=run.metadata.uid,
+                db=runtime.spec.rundb,
+            )
+            self._store_function(runtime, run)
+
+        execution = mlrun.run.MLClientCtx.from_dict(
+            run.to_dict(),
+            runtime._get_db(),
+            autocommit=False,
+            is_api=False,
+            store_run=False,
+        )
+
+        # create task generator (for child runs) from spec
+        task_generator = mlrun.runtimes.generators.get_generator(run.spec, execution)
+        if task_generator:
+            # verify valid task parameters
+            tasks = task_generator.generate(run)
+            for task in tasks:
+                self._validate_run_params(task.spec.parameters)
+
+        # post verifications, store execution in db and run pre run hooks
+        execution.store_run()
+        runtime._pre_run(run, execution)  # hook for runtime specific prep
+
+        last_err = None
+        # If the runtime is nested, it means the hyper-run will run within a single instance of the run.
+        # So while in the API, we consider the hyper-run as a single run, and then in the runtime itself when the
+        # runtime is now a local runtime and therefore `self._is_nested == False`, we run each task as a separate run by
+        # using the task generator
+        # TODO client-server separation might not need the not runtime._is_nested anymore as this executed local func
+        if task_generator and not runtime._is_nested:
+            # multiple runs (based on hyper params or params file)
+            runner = runtime._run_many
+            if hasattr(runtime, "_parallel_run_many") and task_generator.use_parallel():
+                runner = runtime._parallel_run_many
+            results = runner(task_generator, execution, run)
+            mlrun.runtimes.utils.results_to_iter(results, run, execution)
+            result = execution.to_dict()
+            result = runtime._update_run_state(result, task=run)
+
+        else:
+            # single run
+            try:
+                resp = runtime._run(run, execution)
+                result = runtime._update_run_state(resp, task=run)
+            except mlrun.runtimes.base.RunError as err:
+                last_err = err
+                result = runtime._update_run_state(task=run, err=err)
+
+        self._push_notifications(run, runtime)
+
+        # run post run hooks
+        runtime._post_run(result, execution)  # hook for runtime specific cleanup
+
+        return self._wrap_run_result(runtime, result, run, err=last_err)
+
+    def _create_local_function_for_execution(
+        self,
+        runtime: "mlrun.runtimes.BaseRuntime",
+        run: "mlrun.run.RunObject",
+        local_code_path: Optional[str] = None,
+        project: Optional[str] = "",
+        name: Optional[str] = "",
+        workdir: Optional[str] = "",
+        handler: Optional[str] = None,
+    ):
+
+        project = project or runtime.metadata.project
+        function_name = name or runtime.metadata.name
+        command, args = self._resolve_local_code_path(local_code_path)
+        if command:
+            function_name = name or pathlib.Path(command).stem
+
+        meta = mlrun.model.BaseMetadata(function_name, project=project)
+
+        command, loaded_runtime = mlrun.run.load_func_code(
+            command or runtime, workdir, name=name
+        )
+        # loaded_runtime is loaded from runtime or yaml file, if passed a command it should be None,
+        # so we keep the current runtime for enrichment
+        runtime = loaded_runtime or runtime
+        if loaded_runtime:
+            if run:
+                handler = handler or run.spec.handler
+            handler = handler or runtime.spec.default_handler or ""
+            meta = runtime.metadata.copy()
+            meta.name = function_name or meta.name
+            meta.project = project or meta.project
+
+        # if the handler has module prefix force "local" (vs "handler") runtime
+        kind = "local" if isinstance(handler, str) and "." in handler else ""
+        fn = mlrun.new_function(meta.name, command=command, args=args, kind=kind)
+        fn.metadata = meta
+        setattr(fn, "_is_run_local", True)
+        if workdir:
+            fn.spec.workdir = str(workdir)
+
+        fn.spec.allow_empty_resources = runtime.spec.allow_empty_resources
+        if runtime:
+            # copy the code/base-spec to the local function (for the UI and code logging)
+            fn.spec.description = runtime.spec.description
+            fn.spec.build = runtime.spec.build
+
+        run.spec.handler = handler
+        return fn
+
+    @staticmethod
+    def _resolve_local_code_path(local_code_path: str) -> (str, List[str]):
+        command = None
+        args = []
+        if local_code_path:
+            command = local_code_path
+            if command:
+                sp = command.split()
+                # split command and args
+                command = sp[0]
+                if len(sp) > 1:
+                    args = sp[1:]
+        return command, args
+
+    def _push_notifications(
+        self, runobj: "mlrun.run.RunObject", runtime: "mlrun.runtimes.BaseRuntime"
+    ):
+        if not self._run_has_valid_notifications(runobj):
+            return
+        # TODO: add store_notifications API endpoint so we can store notifications pushed from the
+        #       SDK for documentation purposes.
+        # The run is local, so we can assume that watch=True, therefore this code runs
+        # once the run is completed, and we can just push the notifications.
+        # Only push from jupyter, not from the CLI.
+        # "handler" and "dask" kinds are special cases of local runs which don't set local=True
+        if self._is_run_local or runtime.kind in ["handler", "dask"]:
+            mlrun.utils.notifications.NotificationPusher([runobj]).push()

mlrun/launcher/remote.py

@@ -0,0 +1,178 @@
+# Copyright 2023 MLRun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+from typing import Dict, List, Optional, Union
+
+import requests
+
+import mlrun.common.schemas.schedule
+import mlrun.db
+import mlrun.errors
+import mlrun.launcher.client
+import mlrun.run
+import mlrun.runtimes
+import mlrun.runtimes.generators
+import mlrun.utils.clones
+import mlrun.utils.notifications
+from mlrun.utils import logger
+
+
+class ClientRemoteLauncher(mlrun.launcher.client.ClientBaseLauncher):
+    def launch(
+        self,
+        runtime: "mlrun.runtimes.KubejobRuntime",
+        task: Optional[
+            Union["mlrun.run.RunTemplate", "mlrun.run.RunObject", dict]
+        ] = None,
+        handler: Optional[str] = None,
+        name: Optional[str] = "",
+        project: Optional[str] = "",
+        params: Optional[dict] = None,
+        inputs: Optional[Dict[str, str]] = None,
+        out_path: Optional[str] = "",
+        workdir: Optional[str] = "",
+        artifact_path: Optional[str] = "",
+        watch: Optional[bool] = True,
+        schedule: Optional[
+            Union[str, mlrun.common.schemas.schedule.ScheduleCronTrigger]
+        ] = None,
+        hyperparams: Dict[str, list] = None,
+        hyper_param_options: Optional[mlrun.model.HyperParamOptions] = None,
+        verbose: Optional[bool] = None,
+        scrape_metrics: Optional[bool] = None,
+        local_code_path: Optional[str] = None,
+        auto_build: Optional[bool] = None,
+        param_file_secrets: Optional[Dict[str, str]] = None,
+        notifications: Optional[List[mlrun.model.Notification]] = None,
+        returns: Optional[List[Union[str, Dict[str, str]]]] = None,
+    ) -> "mlrun.run.RunObject":
+        self.enrich_runtime(runtime)
+        run = self._create_run_object(task)
+
+        run = self._enrich_run(
+            runtime=runtime,
+            run=run,
+            handler=handler,
+            project_name=project,
+            name=name,
+            params=params,
+            inputs=inputs,
+            returns=returns,
+            hyperparams=hyperparams,
+            hyper_param_options=hyper_param_options,
+            verbose=verbose,
+            scrape_metrics=scrape_metrics,
+            out_path=out_path,
+            artifact_path=artifact_path,
+            workdir=workdir,
+            notifications=notifications,
+        )
+        self._validate_runtime(runtime, run)
+
+        if not runtime.is_deployed():
+            if runtime.spec.build.auto_build or auto_build:
+                logger.info(
+                    "Function is not deployed and auto_build flag is set, starting deploy..."
+                )
+                runtime.deploy(skip_deployed=True, show_on_failure=True)
+
+            else:
+                raise mlrun.errors.MLRunRuntimeError(
+                    "function image is not built/ready, set auto_build=True or use .deploy() method first"
+                )
+
+        if runtime.verbose:
+            logger.info(f"runspec:\n{run.to_yaml()}")
+
+        if "V3IO_USERNAME" in os.environ and "v3io_user" not in run.metadata.labels:
+            run.metadata.labels["v3io_user"] = os.environ.get("V3IO_USERNAME")
+
+        logger.info(
+            "Storing function",
+            name=run.metadata.name,
+            uid=run.metadata.uid,
+            db=runtime.spec.rundb,
+        )
+        self._store_function(runtime, run)
+
+        return self.submit_job(runtime, run, schedule, watch)
+
+    def submit_job(
+        self,
+        runtime: "mlrun.runtimes.KubejobRuntime",
+        run: "mlrun.run.RunObject",
+        schedule: Optional[mlrun.common.schemas.ScheduleCronTrigger] = None,
+        watch: Optional[bool] = None,
+    ):
+        if runtime._secrets:
+            run.spec.secret_sources = runtime._secrets.to_serial()
+        try:
+            db = runtime._get_db()
+            resp = db.submit_job(run, schedule=schedule)
+            if schedule:
+                action = resp.pop("action", "created")
+                logger.info(f"task schedule {action}", **resp)
+                return
+
+        except (requests.HTTPError, Exception) as err:
+            logger.error(f"got remote run err, {mlrun.errors.err_to_str(err)}")
+
+            if isinstance(err, requests.HTTPError):
+                runtime._handle_submit_job_http_error(err)
+
+            result = None
+            # if we got a schedule no reason to do post_run stuff (it purposed to update the run status with error,
+            # but there's no run in case of schedule)
+            if not schedule:
+                result = runtime._update_run_state(
+                    task=run, err=mlrun.errors.err_to_str(err)
+                )
+            return self._wrap_run_result(
+                runtime, result, run, schedule=schedule, err=err
+            )
+
+        if resp:
+            txt = mlrun.runtimes.utils.helpers.get_in(resp, "status.status_text")
+            if txt:
+                logger.info(txt)
+
+        # watch is None only in scenario where we run from pipeline step, in this case we don't want to watch the run
+        # logs too frequently but rather just pull the state of the run from the DB and pull the logs every x seconds
+        # which ideally greater than the pull state interval, this reduces unnecessary load on the API server, as
+        # running a pipeline is mostly not an interactive process which means the logs pulling doesn't need to be pulled
+        # in real time
+        if (
+            watch is None
+            and runtime.kfp
+            and mlrun.mlconf.httpdb.logs.pipelines.pull_state.mode == "enabled"
+        ):
+            state_interval = int(
+                mlrun.mlconf.httpdb.logs.pipelines.pull_state.pull_state_interval
+            )
+            logs_interval = int(
+                mlrun.mlconf.httpdb.logs.pipelines.pull_state.pull_logs_interval
+            )
+            run.wait_for_completion(
+                show_logs=True,
+                sleep=state_interval,
+                logs_interval=logs_interval,
+                raise_on_failure=False,
+            )
+            resp = runtime._get_db_run(run)
+
+        elif watch or runtime.kfp:
+            run.logs(True, runtime._get_db())
+            resp = runtime._get_db_run(run)
+
+        return self._wrap_run_result(runtime, resp, run, schedule=schedule)

mlrun/lists.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -92,7 +92,9 @@ class RunList(list):
 
         return [list_header] + rows
 
-    def to_df(self, flat=False, extend_iterations=False, cache=True):
+    def to_df(
+        self, flat: bool = False, extend_iterations: bool = False, cache: bool = True
+    ) -> pd.DataFrame:
         """convert the run list to a dataframe"""
         if hasattr(self, "_df") and cache:
             return self._df
@@ -179,10 +181,16 @@ class ArtifactList(list):
             "producer": ["producer", "spec.producer"],
             "sources": ["sources", "spec.sources"],
             "labels": ["labels", "metadata.labels"],
+            # important: the uri item must be the last one in this dict since there is no artifact.uri, and we fill it
+            # in the following for loop as the "last_index" in the dict
+            "uri": ["uri", "uri"],
         }
         for artifact in self:
             fields_index = 0 if is_legacy_artifact(artifact) else 1
             row = [get_in(artifact, v[fields_index], "") for k, v in head.items()]
+            artifact_uri = dict_to_artifact(artifact).uri
+            last_index = len(row) - 1
+            row[last_index] = artifact_uri
             rows.append(row)
 
         return [head.keys()] + rows

mlrun/mlutils/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/mlutils/data.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/mlutils/models.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/mlutils/plots.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.