mlrun 1.7.0rc22__py3-none-any.whl → 1.7.0rc28__py3-none-any.whl

mlrun/model_monitoring/stream_processing.py

@@ -66,10 +66,6 @@ class EventStreamProcessor:
         self.parquet_batching_max_events = parquet_batching_max_events
         self.parquet_batching_timeout_secs = parquet_batching_timeout_secs
 
-        self.model_endpoint_store_target = (
-            mlrun.mlconf.model_endpoint_monitoring.store_type
-        )
-
         logger.info(
             "Initializing model monitoring event stream processor",
             parquet_path=self.parquet_path,
@@ -139,7 +135,7 @@ class EventStreamProcessor:
     def apply_monitoring_serving_graph(
         self,
         fn: mlrun.runtimes.ServingRuntime,
-        tsdb_service_provider: typing.Optional[typing.Callable] = None,
+        secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
     ) -> None:
         """
         Apply monitoring serving graph to a given serving function. The following serving graph includes about 4 main
@@ -167,7 +163,8 @@ class EventStreamProcessor:
            using CE, the parquet target path is based on the defined MLRun artifact path.
 
         :param fn: A serving function.
-        :param tsdb_service_provider: An optional callable function that provides the TSDB connection string.
+        :param secret_provider: An optional callable function that provides the connection string from the project
+                                secret.
         """
 
         graph = typing.cast(
@@ -293,7 +290,6 @@ class EventStreamProcessor:
                 name="UpdateEndpoint",
                 after="ProcessBeforeEndpointUpdate",
                 project=self.project,
-                model_endpoint_store_target=self.model_endpoint_store_target,
             )
 
         apply_update_endpoint()
@@ -310,7 +306,10 @@ class EventStreamProcessor:
                 table=self.kv_path,
             )
 
-        if self.model_endpoint_store_target == ModelEndpointTarget.V3IO_NOSQL:
+        store_object = mlrun.model_monitoring.get_store_object(
+            project=self.project, secret_provider=secret_provider
+        )
+        if store_object.type == ModelEndpointTarget.V3IO_NOSQL:
             apply_infer_schema()
 
         # Emits the event in window size of events based on sample_window size (10 by default)
@@ -328,7 +327,7 @@ class EventStreamProcessor:
         # TSDB branch (skip to Prometheus if in CE env)
         if not mlrun.mlconf.is_ce_mode():
             tsdb_connector = mlrun.model_monitoring.get_tsdb_connector(
-                project=self.project, secret_provider=tsdb_service_provider
+                project=self.project, secret_provider=secret_provider
             )
             tsdb_connector.apply_monitoring_stream_steps(graph=graph)
 
@@ -904,7 +903,7 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
 
 
 class UpdateEndpoint(mlrun.feature_store.steps.MapClass):
-    def __init__(self, project: str, model_endpoint_store_target: str, **kwargs):
+    def __init__(self, project: str, **kwargs):
         """
         Update the model endpoint record in the DB. Note that the event at this point includes metadata and stats about
         the average latency and the amount of predictions over time. This data will be used in the monitoring dashboards
@@ -914,7 +913,6 @@ class UpdateEndpoint(mlrun.feature_store.steps.MapClass):
         """
         super().__init__(**kwargs)
         self.project = project
-        self.model_endpoint_store_target = model_endpoint_store_target
 
     def do(self, event: dict):
         # Remove labels from the event

mlrun/model_monitoring/writer.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import json
-from typing import Any, NewType
+from typing import Any, Callable, NewType
 
 import mlrun.common.model_monitoring
 import mlrun.common.schemas
@@ -30,7 +30,7 @@ from mlrun.common.schemas.model_monitoring.constants import (
     WriterEventKind,
 )
 from mlrun.common.schemas.notification import NotificationKind, NotificationSeverity
-from mlrun.model_monitoring.helpers import get_endpoint_record, get_result_instance_fqn
+from mlrun.model_monitoring.helpers import get_result_instance_fqn
 from mlrun.serving.utils import StepToDict
 from mlrun.utils import logger
 from mlrun.utils.notifications.notification_pusher import CustomNotificationPusher
@@ -102,7 +102,11 @@ class ModelMonitoringWriter(StepToDict):
 
     kind = "monitoring_application_stream_pusher"
 
-    def __init__(self, project: str, tsdb_secret_provider=None) -> None:
+    def __init__(
+        self,
+        project: str,
+        secret_provider: Callable = None,
+    ) -> None:
         self.project = project
         self.name = project  # required for the deployment process
 
@@ -111,10 +115,10 @@ class ModelMonitoringWriter(StepToDict):
         )
 
         self._app_result_store = mlrun.model_monitoring.get_store_object(
-            project=self.project
+            project=self.project, secret_provider=secret_provider
         )
         self._tsdb_connector = mlrun.model_monitoring.get_tsdb_connector(
-            project=self.project, secret_provider=tsdb_secret_provider
+            project=self.project, secret_provider=secret_provider
         )
         self._endpoints_records = {}
 
@@ -149,11 +153,7 @@ class ModelMonitoringWriter(StepToDict):
         result_kind: int, result_status: int
     ) -> alert_objects.EventKind:
         """Generate the required Event Kind format for the alerting system"""
-        if result_kind == ResultKindApp.custom.value:
-            # Custom kind is represented as an anomaly detection
-            event_kind = "mm_app_anomaly"
-        else:
-            event_kind = ResultKindApp(value=result_kind).name
+        event_kind = ResultKindApp(value=result_kind).name
 
         if result_status == ResultStatusApp.detected.value:
             event_kind = f"{event_kind}_detected"
@@ -223,7 +223,7 @@ class ModelMonitoringWriter(StepToDict):
             endpoint_id = event[WriterEvent.ENDPOINT_ID]
             endpoint_record = self._endpoints_records.setdefault(
                 endpoint_id,
-                get_endpoint_record(project=self.project, endpoint_id=endpoint_id),
+                self._app_result_store.get_model_endpoint(endpoint_id=endpoint_id),
             )
             event_value = {
                 "app_name": event[WriterEvent.APPLICATION_NAME],

mlrun/package/__init__.py

@@ -12,19 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-"""
-MLRun package enables fully-automated experiment and pipeline tracking and reproducibility, and easy passing of
-python objects between remote jobs, while not requiring any form of editing to the actual function original code.
-Simply set the function code in a project and run it, MLRun takes care of the rest.
-
-MLRun uses packagers: classes that perform 2 tasks:
-
-#. **Parsing inputs** - automatically cast the runtime's inputs (user's input passed to the function via
-the ``inputs`` parameter of the ``run`` method) to the relevant hinted type.  (Does not require handling of data items.)
-#. **Logging outputs** - automatically save, log, and upload the function's returned objects by the provided
-log hints (user's input passed to the function via the ``returns`` parameter of the ``run`` method).
-(Does not require handling of files and artifacts.)
-"""
+
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
 import functools

mlrun/package/packagers/__init__.py

@@ -12,12 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-r"""
-MLRun comes with the following list of modules, out of the box. All of the packagers listed here
-use the implementation of :ref:`DefaultPackager <mlrun.package.packagers.default\_packager.DefaultPackager>` and are
-available by default at the start of each run.
-"""
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
+# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 from .default_packager import DefaultPackager
 from .numpy_packagers import NumPySupportedFormat

mlrun/projects/pipelines.py

@@ -22,8 +22,7 @@ import uuid
 
 import mlrun_pipelines.common.models
 import mlrun_pipelines.patcher
-from kfp.compiler import compiler
-from mlrun_pipelines.helpers import new_pipe_metadata
+import mlrun_pipelines.utils
 
 import mlrun
 import mlrun.common.runtimes.constants
@@ -220,9 +219,10 @@ class _PipelineContext:
         force_run_local = mlrun.mlconf.force_run_local
         if force_run_local is None or force_run_local == "auto":
             force_run_local = not mlrun.mlconf.is_api_running_on_k8s()
-            kfp_url = mlrun.mlconf.resolve_kfp_url()
-            if not kfp_url:
+            if not mlrun.mlconf.kfp_url:
+                logger.debug("Kubeflow pipeline URL is not set, running locally")
                 force_run_local = True
+
         if self.workflow:
             force_run_local = force_run_local or self.workflow.run_local
 
@@ -502,13 +502,14 @@ class _KFPRunner(_PipelineRunner):
             functions,
             secrets=project._secrets,
         )
-        artifact_path = artifact_path or project.spec.artifact_path
-
-        conf = new_pipe_metadata(
-            artifact_path=artifact_path,
+        mlrun_pipelines.utils.compile_pipeline(
+            artifact_path=artifact_path or project.spec.artifact_path,
             cleanup_ttl=workflow_spec.cleanup_ttl,
+            ops=None,
+            pipeline=pipeline,
+            pipe_file=target,
+            type_check=True,
         )
-        compiler.Compiler().compile(pipeline, target, pipeline_conf=conf)
         workflow_spec.clear_tmp()
         pipeline_context.clear()
 

mlrun/projects/project.py

@@ -51,6 +51,7 @@ import mlrun.runtimes.nuclio.api_gateway
 import mlrun.runtimes.pod
 import mlrun.runtimes.utils
 import mlrun.serving
+import mlrun.utils
 import mlrun.utils.regex
 from mlrun.alerts.alert import AlertConfig
 from mlrun.common.schemas.alert import AlertTemplate
@@ -713,7 +714,8 @@ def _project_instance_from_struct(struct, name, allow_cross_project):
     name_from_struct = struct.get("metadata", {}).get("name", "")
     if name and name_from_struct and name_from_struct != name:
         error_message = (
-            f"project name mismatch, {name_from_struct} != {name}, please do one of the following:\n"
+            f"Project name mismatch, {name_from_struct} != {name}, project is loaded from {name_from_struct} "
+            f"project yaml. To prevent/allow this, you can take one of the following actions:\n"
             "1. Set the `allow_cross_project=True` when loading the project.\n"
             f"2. Delete the existing project yaml, or ensure its name is equal to {name}.\n"
             "3. Use different project context dir."
@@ -721,14 +723,14 @@ def _project_instance_from_struct(struct, name, allow_cross_project):
 
         if allow_cross_project is None:
             # TODO: Remove this warning in version 1.9.0 and also fix cli to support allow_cross_project
-            logger.warn(
-                "Project name is different than specified on its project yaml."
-                "You should fix it until version 1.9.0",
-                description=error_message,
+            warnings.warn(
+                f"Project {name=} is different than specified on the context's project yaml. "
+                "This behavior is deprecated and will not be supported in version 1.9.0."
             )
+            logger.warn(error_message)
         elif allow_cross_project:
-            logger.warn(
-                "Project name is different than specified on its project yaml. Overriding.",
+            logger.debug(
+                "Project name is different than specified on the context's project yaml. Overriding.",
                 existing_name=name_from_struct,
                 overriding_name=name,
             )
@@ -993,15 +995,29 @@ class ProjectSpec(ModelObj):
 
         artifacts_dict = {}
         for artifact in artifacts:
-            if not isinstance(artifact, dict) and not hasattr(artifact, "to_dict"):
+            invalid_object_type = not isinstance(artifact, dict) and not hasattr(
+                artifact, "to_dict"
+            )
+            is_artifact_model = not isinstance(artifact, dict) and hasattr(
+                artifact, "to_dict"
+            )
+
+            if invalid_object_type:
                 raise ValueError("artifacts must be a dict or class")
-            if isinstance(artifact, dict):
-                key = artifact.get("metadata", {}).get("key", "")
-                if not key:
-                    raise ValueError('artifacts "metadata.key" must be specified')
-            else:
+            elif is_artifact_model:
                 key = artifact.key
                 artifact = artifact.to_dict()
+            else:  # artifact is a dict
+                # imported/legacy artifacts don't have metadata,spec,status fields
+                key_field = (
+                    "key"
+                    if _is_imported_artifact(artifact)
+                    or mlrun.utils.is_legacy_artifact(artifact)
+                    else "metadata.key"
+                )
+                key = mlrun.utils.get_in(artifact, key_field, "")
+                if not key:
+                    raise ValueError(f'artifacts "{key_field}" must be specified')
 
             artifacts_dict[key] = artifact
 
@@ -2116,6 +2132,8 @@ class MlrunProject(ModelObj):
         *,
         deploy_histogram_data_drift_app: bool = True,
         wait_for_deployment: bool = False,
+        rebuild_images: bool = False,
+        fetch_credentials_from_sys_config: bool = False,
     ) -> None:
         """
         Deploy model monitoring application controller, writer and stream functions.
@@ -2125,16 +2143,18 @@ class MlrunProject(ModelObj):
         The stream function goal is to monitor the log of the data stream. It is triggered when a new log entry
         is detected. It processes the new events into statistics that are then written to statistics databases.
 
-        :param default_controller_image:        Deprecated.
-        :param base_period:                     The time period in minutes in which the model monitoring controller
-                                                function is triggered. By default, the base period is 10 minutes.
-        :param image:                           The image of the model monitoring controller, writer, monitoring
-                                                stream & histogram data drift functions, which are real time nuclio
-                                                functions. By default, the image is mlrun/mlrun.
-        :param deploy_histogram_data_drift_app: If true, deploy the default histogram-based data drift application.
-        :param wait_for_deployment:             If true, return only after the deployment is done on the backend.
-                                                Otherwise, deploy the model monitoring infrastructure on the
-                                                background, including the histogram data drift app if selected.
+        :param default_controller_image:          Deprecated.
+        :param base_period:                       The time period in minutes in which the model monitoring controller
+                                                  function is triggered. By default, the base period is 10 minutes.
+        :param image:                             The image of the model monitoring controller, writer, monitoring
+                                                  stream & histogram data drift functions, which are real time nuclio
+                                                  functions. By default, the image is mlrun/mlrun.
+        :param deploy_histogram_data_drift_app:   If true, deploy the default histogram-based data drift application.
+        :param wait_for_deployment:               If true, return only after the deployment is done on the backend.
+                                                  Otherwise, deploy the model monitoring infrastructure on the
+                                                  background, including the histogram data drift app if selected.
+        :param rebuild_images:                    If true, force rebuild of model monitoring infrastructure images.
+        :param fetch_credentials_from_sys_config: If true, fetch the credentials from the system configuration.
         """
         if default_controller_image != "mlrun/mlrun":
             # TODO: Remove this in 1.9.0
@@ -2150,6 +2170,8 @@ class MlrunProject(ModelObj):
             image=image,
             base_period=base_period,
             deploy_histogram_data_drift_app=deploy_histogram_data_drift_app,
+            rebuild_images=rebuild_images,
+            fetch_credentials_from_sys_config=fetch_credentials_from_sys_config,
         )
 
         if wait_for_deployment:
@@ -2324,7 +2346,8 @@ class MlrunProject(ModelObj):
                                     Default: job
         :param image:               Docker image to be used, can also be specified in the function object/yaml
         :param handler:             Default function handler to invoke (can only be set with .py/.ipynb files)
-        :param with_repo:           Add (clone) the current repo to the build source
+        :param with_repo:           Add (clone) the current repo to the build source - use when the function code is in
+                                    the project repo (project.spec.source).
         :param tag:                 Function version tag to set (none for current or 'latest')
                                     Specifying a tag as a parameter will update the project's tagged function
                                     (myfunc:v1) and the untagged function (myfunc)
@@ -2471,25 +2494,17 @@ class MlrunProject(ModelObj):
         self.spec.remove_function(name)
 
     def remove_model_monitoring_function(self, name: Union[str, list[str]]):
-        """remove the specified model-monitoring-app function/s from the project spec
+        """delete the specified model-monitoring-app function/s
 
         :param name: name of the model-monitoring-function/s (under the project)
         """
-        names = name if isinstance(name, list) else [name]
-        for func_name in names:
-            function = self.get_function(key=func_name)
-            if (
-                function.metadata.labels.get(mm_constants.ModelMonitoringAppLabel.KEY)
-                == mm_constants.ModelMonitoringAppLabel.VAL
-            ):
-                self.remove_function(name=func_name)
-                logger.info(
-                    f"{func_name} function has been removed from {self.name} project"
-                )
-            else:
-                raise logger.warn(
-                    f"There is no model monitoring function with {func_name} name"
-                )
+        # TODO: Remove this in 1.9.0
+        warnings.warn(
+            "'remove_model_monitoring_function' is deprecated and will be removed in 1.9.0. "
+            "Please use `delete_model_monitoring_function` instead.",
+            FutureWarning,
+        )
+        self.delete_model_monitoring_function(name)
 
     def delete_model_monitoring_function(self, name: Union[str, list[str]]):
         """delete the specified model-monitoring-app function/s
@@ -3191,48 +3206,44 @@ class MlrunProject(ModelObj):
         stream_path: Optional[str] = None,
         tsdb_connection: Optional[str] = None,
     ):
-        """Set the credentials that will be used by the project's model monitoring
-        infrastructure functions.
-
-        :param access_key:                Model Monitoring access key for managing user permissions
-        :param endpoint_store_connection: Endpoint store connection string
-        :param stream_path:               Path to the model monitoring stream
-        :param tsdb_connection:           Connection string to the time series database
         """
-
-        secrets_dict = {}
-        if access_key:
-            secrets_dict[
-                mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ACCESS_KEY
-            ] = access_key
-
-        if endpoint_store_connection:
-            secrets_dict[
-                mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ENDPOINT_STORE_CONNECTION
-            ] = endpoint_store_connection
-
-        if stream_path:
-            if stream_path.startswith("kafka://") and "?topic" in stream_path:
-                raise mlrun.errors.MLRunInvalidArgumentError(
-                    "Custom kafka topic is not allowed"
-                )
-            secrets_dict[
-                mlrun.common.schemas.model_monitoring.ProjectSecretKeys.STREAM_PATH
-            ] = stream_path
-
-        if tsdb_connection:
-            if not tsdb_connection.startswith("taosws://"):
-                raise mlrun.errors.MLRunInvalidArgumentError(
-                    "Currently only TDEngine websocket connection is supported for non-v3io TSDB,"
-                    "please provide a full URL (e.g. taosws://user:password@host:port)"
-                )
-            secrets_dict[
-                mlrun.common.schemas.model_monitoring.ProjectSecretKeys.TSDB_CONNECTION
-            ] = tsdb_connection
-
-        self.set_secrets(
-            secrets=secrets_dict,
-            provider=mlrun.common.schemas.SecretProviderName.kubernetes,
+        Set the credentials that will be used by the project's model monitoring
+        infrastructure functions. Important to note that you have to set the credentials before deploying any
+        model monitoring or serving function.
+
+        :param access_key:                Model Monitoring access key for managing user permissions.
+        :param endpoint_store_connection: Endpoint store connection string. By default, None.
+                                          Options:
+                                          1. None, will be set from the system configuration.
+                                          2. v3io - for v3io endpoint store,
+                                             pass `v3io` and the system will generate the exact path.
+                                          3. MySQL/SQLite - for SQL endpoint store, please provide full
+                                             connection string, for example
+                                             mysql+pymysql://<username>:<password>@<host>:<port>/<db_name>
+        :param stream_path:               Path to the model monitoring stream. By default, None.
+                                          Options:
+                                          1. None, will be set from the system configuration.
+                                          2. v3io - for v3io stream,
+                                             pass `v3io` and the system will generate the exact path.
+                                          3. Kafka - for Kafka stream, please provide full connection string without
+                                             custom topic, for example kafka://<some_kafka_broker>:<port>.
+        :param tsdb_connection:           Connection string to the time series database. By default, None.
+                                          Options:
+                                          1. None, will be set from the system configuration.
+                                          2. v3io - for v3io stream,
+                                             pass `v3io` and the system will generate the exact path.
+                                          3. TDEngine - for TDEngine tsdb, please provide full websocket connection URL,
+                                             for example taosws://<username>:<password>@<host>:<port>.
+        """
+        db = mlrun.db.get_run_db(secrets=self._secrets)
+        db.set_model_monitoring_credentials(
+            project=self.name,
+            credentials={
+                "access_key": access_key,
+                "endpoint_store_connection": endpoint_store_connection,
+                "stream_path": stream_path,
+                "tsdb_connection": tsdb_connection,
+            },
         )
 
     def run_function(
@@ -3649,6 +3660,7 @@ class MlrunProject(ModelObj):
         kind: str = None,
         category: typing.Union[str, mlrun.common.schemas.ArtifactCategories] = None,
         tree: str = None,
+        limit: int = None,
     ) -> mlrun.lists.ArtifactList:
         """List artifacts filtered by various parameters.
 
@@ -3678,6 +3690,7 @@ class MlrunProject(ModelObj):
         :param kind: Return artifacts of the requested kind.
         :param category: Return artifacts of the requested category.
         :param tree: Return artifacts of the requested tree.
+        :param limit: Maximum number of artifacts to return.
         """
         db = mlrun.db.get_run_db(secrets=self._secrets)
         return db.list_artifacts(
@@ -3692,6 +3705,7 @@ class MlrunProject(ModelObj):
             kind=kind,
             category=category,
             tree=tree,
+            limit=limit,
         )
 
     def list_models(

mlrun/render.py

@@ -283,9 +283,14 @@ function copyToClipboard(fld) {
 }
 function expandPanel(el) {
   const panelName = "#" + el.getAttribute('paneName');
-  console.log(el.title);
 
-  document.querySelector(panelName + "-title").innerHTML = el.title
+  // Get the base URL of the current notebook
+  var baseUrl = window.location.origin;
+
+  // Construct the full URL
+  var fullUrl = new URL(el.title, baseUrl).href;
+
+  document.querySelector(panelName + "-title").innerHTML = fullUrl
   iframe = document.querySelector(panelName + "-body");
 
   const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}
@@ -299,7 +304,7 @@ function expandPanel(el) {
   }
 
   function reqListener () {
-    if (el.title.endsWith(".csv")) {
+    if (fullUrl.endsWith(".csv")) {
       iframe.setAttribute("srcdoc", tblcss + csvToHtmlTable(this.responseText));
     } else {
       iframe.setAttribute("srcdoc", this.responseText);
@@ -309,11 +314,11 @@ function expandPanel(el) {
 
   const oReq = new XMLHttpRequest();
   oReq.addEventListener("load", reqListener);
-  oReq.open("GET", el.title);
+  oReq.open("GET", fullUrl);
   oReq.send();
 
 
-  //iframe.src = el.title;
+  //iframe.src = fullUrl;
   const resultPane = document.querySelector(panelName + "-pane");
   if (resultPane.classList.contains("hidden")) {
     resultPane.classList.remove("hidden");

mlrun/run.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import importlib.util as imputil
 import json
 import os
@@ -28,9 +29,9 @@ from typing import Optional, Union
 
 import nuclio
 import yaml
-from kfp import Client
 from mlrun_pipelines.common.models import RunStatuses
 from mlrun_pipelines.common.ops import format_summary_from_kfp_run, show_kfp_run
+from mlrun_pipelines.utils import get_client
 
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.formatters
@@ -62,11 +63,11 @@ from .runtimes.funcdoc import update_function_entry_points
 from .runtimes.nuclio.application import ApplicationRuntime
 from .runtimes.utils import add_code_metadata, global_context
 from .utils import (
+    RunKeys,
     extend_hub_uri_if_needed,
     get_in,
     logger,
     retry_until_successful,
-    run_keys,
     update_in,
 )
 
@@ -279,7 +280,7 @@ def get_or_create_ctx(
             artifact_path = mlrun.utils.helpers.template_artifact_path(
                 mlconf.artifact_path, project or mlconf.default_project
             )
-            update_in(newspec, ["spec", run_keys.output_path], artifact_path)
+            update_in(newspec, ["spec", RunKeys.output_path], artifact_path)
 
     newspec.setdefault("metadata", {})
     update_in(newspec, "metadata.name", name, replace=False)
@@ -293,10 +294,14 @@ def get_or_create_ctx(
     newspec["metadata"]["project"] = (
         newspec["metadata"].get("project") or project or mlconf.default_project
     )
+
     newspec["metadata"].setdefault("labels", {})
-    newspec["metadata"]["labels"] = {
-        mlrun_constants.MLRunInternalLabels.kind: RuntimeKinds.local
-    }
+
+    # This function can also be called as a local run if it is not called within a function.
+    # It will create a local run, and the run kind must be local by default.
+    newspec["metadata"]["labels"].setdefault(
+        mlrun_constants.MLRunInternalLabels.kind, RuntimeKinds.local
+    )
 
     ctx = MLClientCtx.from_dict(
         newspec, rundb=out, autocommit=autocommit, tmp=tmp, host=socket.gethostname()
@@ -948,7 +953,7 @@ def wait_for_pipeline_completion(
             _wait_for_pipeline_completion,
         )
     else:
-        client = Client(namespace=namespace)
+        client = get_client(namespace=namespace)
         resp = client.wait_for_run_completion(run_id, timeout)
         if resp:
             resp = resp.to_dict()
@@ -1009,7 +1014,7 @@ def get_pipeline(
         )
 
     else:
-        client = Client(namespace=namespace)
+        client = get_client(namespace=namespace)
         resp = client.get_run(run_id)
         if resp:
             resp = resp.to_dict()

mlrun/runtimes/base.py

@@ -426,13 +426,19 @@ class BaseRuntime(ModelObj):
             reset_on_run=reset_on_run,
         )
 
-    def _get_db_run(self, task: RunObject = None):
+    def _get_db_run(
+        self,
+        task: RunObject = None,
+        run_format: mlrun.common.formatters.RunFormat = mlrun.common.formatters.RunFormat.full,
+    ):
         if self._get_db() and task:
             project = task.metadata.project
             uid = task.metadata.uid
             iter = task.metadata.iteration
             try:
-                return self._get_db().read_run(uid, project, iter=iter)
+                return self._get_db().read_run(
+                    uid, project, iter=iter, format_=run_format
+                )
             except mlrun.db.RunDBError:
                 return None
         if task:
@@ -549,13 +555,14 @@ class BaseRuntime(ModelObj):
         self,
         resp: dict = None,
         task: RunObject = None,
-        err=None,
+        err: Union[Exception, str] = None,
+        run_format: mlrun.common.formatters.RunFormat = mlrun.common.formatters.RunFormat.full,
     ) -> typing.Optional[dict]:
         """update the task state in the DB"""
         was_none = False
         if resp is None and task:
             was_none = True
-            resp = self._get_db_run(task)
+            resp = self._get_db_run(task, run_format)
 
             if not resp:
                 self.store_run(task)