mlrun 1.4.0rc25__py3-none-any.whl → 1.5.0rc2__py3-none-any.whl

mlrun/projects/project.py

@@ -38,8 +38,7 @@ import requests
 import yaml
 from deprecated import deprecated
 
-import mlrun.common.model_monitoring as model_monitoring_constants
-import mlrun.common.schemas
+import mlrun.common.schemas.model_monitoring
 import mlrun.db
 import mlrun.errors
 import mlrun.runtimes
@@ -376,7 +375,7 @@ def load_project(
         except Exception:
             pass
 
-    to_save = save and mlrun.mlconf.dbpath
+    to_save = bool(save and mlrun.mlconf.dbpath)
     if to_save:
         project.save()
 
@@ -1246,6 +1245,7 @@ class MlrunProject(ModelObj):
         handler=None,
         schedule: typing.Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
         ttl=None,
+        image: str = None,
         **args,
     ):
         """add or update a workflow, specify a name and the code path
@@ -1261,6 +1261,7 @@ class MlrunProject(ModelObj):
                               see this link for help:
                               https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron
         :param ttl:           pipeline ttl in secs (after that the pods will be removed)
+        :param image:         image for workflow runner job, only for scheduled and remote workflows
         :param args:          argument values (key=value, ..)
         """
 
@@ -1300,6 +1301,8 @@ class MlrunProject(ModelObj):
         workflow["schedule"] = schedule
         if ttl:
             workflow["ttl"] = ttl
+        if image:
+            workflow["image"] = image
         self.spec.set_workflow(name, workflow)
 
     def set_artifact(
@@ -2199,15 +2202,12 @@ class MlrunProject(ModelObj):
         sync: bool = False,
         watch: bool = False,
         dirty: bool = False,
-        # TODO: deprecated, remove in 1.5.0
-        ttl: int = None,
         engine: str = None,
         local: bool = None,
         schedule: typing.Union[
             str, mlrun.common.schemas.ScheduleCronTrigger, bool
         ] = None,
         timeout: int = None,
-        overwrite: bool = False,
         source: str = None,
         cleanup_ttl: int = None,
     ) -> _PipelineRunStatus:
@@ -2227,8 +2227,6 @@ class MlrunProject(ModelObj):
         :param sync:      force functions sync before run
         :param watch:     wait for pipeline completion
         :param dirty:     allow running the workflow when the git repo is dirty
-        :param ttl:       pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
-                          workflow and all its resources are deleted) (deprecated, use cleanup_ttl instead)
         :param engine:    workflow engine running the workflow.
                           supported values are 'kfp' (default), 'local' or 'remote'.
                           for setting engine for remote running use 'remote:local' or 'remote:kfp'.
@@ -2239,8 +2237,6 @@ class MlrunProject(ModelObj):
                           https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron
                           for using the pre-defined workflow's schedule, set `schedule=True`
         :param timeout:   timeout in seconds to wait for pipeline completion (watch will be activated)
-        :param overwrite: (deprecated) replacing the schedule of the same workflow (under the same name) if exists
-                          with the new one.
         :param source:    remote source to use instead of the actual `project.spec.source` (used when engine is remote).
                           for other engines the source is to validate that the code is up-to-date
         :param cleanup_ttl:
@@ -2249,22 +2245,6 @@ class MlrunProject(ModelObj):
         :returns: run id
         """
 
-        if ttl:
-            warnings.warn(
-                "'ttl' is deprecated, use 'cleanup_ttl' instead. "
-                "This will be removed in 1.5.0",
-                # TODO: Remove this in 1.5.0
-                FutureWarning,
-            )
-
-        if overwrite:
-            warnings.warn(
-                "'overwrite' is deprecated, running a schedule is now an upsert operation. "
-                "This will be removed in 1.5.0",
-                # TODO: Remove this in 1.5.0
-                FutureWarning,
-            )
-
         arguments = arguments or {}
         need_repo = self.spec._need_repo()
         if self.spec.repo and self.spec.repo.is_dirty():
@@ -2297,9 +2277,7 @@ class MlrunProject(ModelObj):
         else:
             workflow_spec = self.spec._workflows[name].copy()
             workflow_spec.merge_args(arguments)
-        workflow_spec.cleanup_ttl = (
-            cleanup_ttl or ttl or workflow_spec.cleanup_ttl or workflow_spec.ttl
-        )
+        workflow_spec.cleanup_ttl = cleanup_ttl or workflow_spec.cleanup_ttl
         workflow_spec.run_local = local
 
         name = f"{self.metadata.name}-{name}" if name else self.metadata.name
@@ -2372,11 +2350,6 @@ class MlrunProject(ModelObj):
         expected_statuses=None,
         notifiers: CustomNotificationPusher = None,
     ):
-        warnings.warn(
-            "This is deprecated in 1.3.0, and will be removed in 1.5.0. "
-            "Use `timeout` parameter in `project.run()` method instead",
-            FutureWarning,
-        )
         return run._engine.get_run_status(
             project=self,
             run=run,
@@ -2497,12 +2470,12 @@ class MlrunProject(ModelObj):
         secrets_dict = {}
         if access_key:
             secrets_dict[
-                model_monitoring_constants.ProjectSecretKeys.ACCESS_KEY
+                mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ACCESS_KEY
             ] = access_key
 
         if endpoint_store_connection:
             secrets_dict[
-                model_monitoring_constants.ProjectSecretKeys.ENDPOINT_STORE_CONNECTION
+                mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ENDPOINT_STORE_CONNECTION
             ] = endpoint_store_connection
 
         if stream_path:
@@ -2511,7 +2484,7 @@ class MlrunProject(ModelObj):
                     "Custom kafka topic is not allowed"
                 )
             secrets_dict[
-                model_monitoring_constants.ProjectSecretKeys.STREAM_PATH
+                mlrun.common.schemas.model_monitoring.ProjectSecretKeys.STREAM_PATH
             ] = stream_path
 
         self.set_secrets(

mlrun/render.py

@@ -121,7 +121,7 @@ def artifacts_html(
     html = ""
 
     for artifact in artifacts:
-        # TODO: remove this in 1.5.0 once we no longer support legacy format
+        # TODO: remove this in 1.6.0 once we no longer support legacy format
         if mlrun.utils.is_legacy_artifact(artifact):
             attribute_value = artifact.get(attribute_name)
         else:

mlrun/run.py

@@ -19,7 +19,6 @@ import socket
 import tempfile
 import time
 import uuid
-import warnings
 from base64 import b64decode
 from copy import deepcopy
 from os import environ, makedirs, path
@@ -36,9 +35,9 @@ import mlrun.errors
 import mlrun.utils.helpers
 from mlrun.kfpops import format_summary_from_kfp_run, show_kfp_run
 
+from .common.helpers import parse_versioned_object_uri
 from .config import config as mlconf
 from .datastore import store_manager
-from .db import get_or_set_dburl, get_run_db
 from .errors import MLRunInvalidArgumentError, MLRunTimeoutError
 from .execution import MLClientCtx
 from .model import BaseMetadata, RunObject, RunTemplate
@@ -63,8 +62,6 @@ from .utils import (
     extend_hub_uri_if_needed,
     get_in,
     logger,
-    new_pipe_metadata,
-    parse_versioned_object_uri,
     retry_until_successful,
     run_keys,
     update_in,
@@ -476,7 +473,7 @@ def import_function(url="", secrets=None, db="", project=None, new_name=None):
     if url.startswith("db://"):
         url = url[5:]
         _project, name, tag, hash_key = parse_versioned_object_uri(url)
-        db = get_run_db(db or get_or_set_dburl(), secrets=secrets)
+        db = mlrun.db.get_run_db(db or mlrun.db.get_or_set_dburl(), secrets=secrets)
         runtime = db.get_function(name, _project, tag, hash_key)
         if not runtime:
             raise KeyError(f"function {name}:{tag} not found in the DB")
@@ -860,6 +857,12 @@ def code_to_function(
         kind=subkind,
         ignored_tags=ignored_tags,
     )
+    spec["spec"]["env"].append(
+        {
+            "name": "MLRUN_HTTPDB__NUCLIO__EXPLICIT_ACK",
+            "value": mlrun.mlconf.is_explicit_ack(),
+        }
+    )
     spec_kind = get_in(spec, "kind", "")
     if not kind and spec_kind not in ["", "Function"]:
         kind = spec_kind.lower()
@@ -943,125 +946,6 @@ def code_to_function(
     return r
 
 
-@deprecated(
-    version="1.3.0",
-    reason="'run_pipeline' will be removed in 1.5.0, use 'project.run' instead",
-    category=FutureWarning,
-)
-def run_pipeline(
-    pipeline,
-    arguments=None,
-    project=None,
-    experiment=None,
-    run=None,
-    namespace=None,
-    artifact_path=None,
-    ops=None,
-    url=None,
-    # TODO: deprecated, remove in 1.5.0
-    ttl=None,
-    remote: bool = True,
-    cleanup_ttl=None,
-):
-    """
-    remote KubeFlow pipeline execution
-
-    Submit a workflow task to KFP via mlrun API service
-
-    :param pipeline:   KFP pipeline function or path to .yaml/.zip pipeline file
-    :param arguments:  pipeline arguments
-    :param project:    name of project
-    :param experiment: experiment name
-    :param run:        optional, run name
-    :param namespace:  Kubernetes namespace (if not using default)
-    :param url:        optional, url to mlrun API service
-    :param artifact_path:  target location/url for mlrun artifacts
-    :param ops:        additional operators (.apply() to all pipeline functions)
-    :param ttl:        pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
-                       workflow and all its resources are deleted) (deprecated, use cleanup_ttl instead)
-    :param remote:     read kfp data from mlrun service (default=True). Run pipeline from local kfp data (remote=False)
-      is deprecated. Should not be used
-    :param cleanup_ttl:
-                       pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
-                       workflow and all its resources are deleted)
-
-    :returns: kubeflow pipeline id
-    """
-    if ttl:
-        warnings.warn(
-            "'ttl' is deprecated, use 'cleanup_ttl' instead. "
-            "This will be removed in 1.5.0",
-            # TODO: Remove this in 1.5.0
-            FutureWarning,
-        )
-
-    artifact_path = artifact_path or mlconf.artifact_path
-    project = project or mlconf.default_project
-    artifact_path = mlrun.utils.helpers.fill_artifact_path_template(
-        artifact_path, project or mlconf.default_project
-    )
-    if artifact_path and "{{run.uid}}" in artifact_path:
-        artifact_path.replace("{{run.uid}}", "{{workflow.uid}}")
-    if not artifact_path:
-        raise ValueError("artifact path was not specified")
-
-    namespace = namespace or mlconf.namespace
-    arguments = arguments or {}
-
-    if remote or url:
-        from .projects.pipelines import WorkflowSpec, pipeline_context
-
-        clear_pipeline_context = False
-        # if pipeline_context.workflow isn't set it means the `run_pipeline` method was called directly
-        # so to make sure the pipeline and functions inside are being run in the KFP pipeline we set the pipeline
-        # context with KFP engine
-        if not pipeline_context.workflow:
-            workflow_spec = WorkflowSpec(engine="kfp")
-            pipeline_context.set(pipeline_context.project, workflow=workflow_spec)
-            clear_pipeline_context = True
-
-        pipeline_run_id = _run_pipeline(
-            pipeline=pipeline,
-            arguments=arguments,
-            project=project,
-            experiment=experiment,
-            run=run,
-            namespace=namespace,
-            artifact_path=artifact_path,
-            ops=ops,
-            url=url,
-            cleanup_ttl=cleanup_ttl or ttl,
-        )
-
-        if clear_pipeline_context:
-            pipeline_context.clear()
-
-    # this shouldn't be used, keeping for backwards compatibility until the entire method is deprecated
-    else:
-        client = Client(namespace=namespace)
-        if isinstance(pipeline, str):
-            experiment = client.create_experiment(name=experiment)
-            run_result = client.run_pipeline(
-                experiment.id, run, pipeline, params=arguments
-            )
-        else:
-            conf = new_pipe_metadata(
-                artifact_path=artifact_path, cleanup_ttl=ttl, op_transformers=ops
-            )
-            run_result = client.create_run_from_pipeline_func(
-                pipeline,
-                arguments,
-                run_name=run,
-                experiment_name=experiment,
-                pipeline_conf=conf,
-            )
-
-        pipeline_run_id = run_result.run_id
-        logger.info(f"Pipeline run id={id}, check UI for progress")
-
-    return pipeline_run_id
-
-
 def _run_pipeline(
     pipeline,
     arguments=None,

mlrun/runtimes/__init__.py

@@ -30,22 +30,17 @@ from mlrun.runtimes.utils import (
     resolve_spark_operator_version,
 )
 
-from .base import BaseRuntime, BaseRuntimeHandler, RunError, RuntimeClassMode  # noqa
+from .base import BaseRuntime, RunError, RuntimeClassMode  # noqa
 from .constants import MPIJobCRDVersions
-from .daskjob import DaskCluster, DaskRuntimeHandler, get_dask_resource  # noqa
+from .daskjob import DaskCluster, get_dask_resource  # noqa
 from .function import RemoteRuntime
-from .kubejob import KubejobRuntime, KubeRuntimeHandler  # noqa
+from .kubejob import KubejobRuntime  # noqa
 from .local import HandlerRuntime, LocalRuntime  # noqa
-from .mpijob import (  # noqa
-    MpiRuntimeV1,
-    MpiRuntimeV1Alpha1,
-    MpiV1Alpha1RuntimeHandler,
-    MpiV1RuntimeHandler,
-)
+from .mpijob import MpiRuntimeV1, MpiRuntimeV1Alpha1  # noqa
 from .nuclio import nuclio_init_hook
-from .remotesparkjob import RemoteSparkRuntime, RemoteSparkRuntimeHandler
+from .remotesparkjob import RemoteSparkRuntime
 from .serving import ServingRuntime, new_v2_model_server
-from .sparkjob import Spark3Runtime, SparkRuntimeHandler
+from .sparkjob import Spark3Runtime
 
 # for legacy imports (MLModelServer moved from here to /serving)
 from ..serving import MLModelServer, new_v1_model_server  # noqa isort: skip
@@ -216,37 +211,6 @@ class RuntimeKinds(object):
 
 runtime_resources_map = {RuntimeKinds.dask: get_dask_resource()}
 
-runtime_handler_instances_cache = {}
-
-
-def get_runtime_handler(kind: str) -> BaseRuntimeHandler:
-    global runtime_handler_instances_cache
-    if kind == RuntimeKinds.mpijob:
-        mpijob_crd_version = resolve_mpijob_crd_version()
-        crd_version_to_runtime_handler_class = {
-            MPIJobCRDVersions.v1alpha1: MpiV1Alpha1RuntimeHandler,
-            MPIJobCRDVersions.v1: MpiV1RuntimeHandler,
-        }
-        runtime_handler_class = crd_version_to_runtime_handler_class[mpijob_crd_version]
-        if not runtime_handler_instances_cache.setdefault(RuntimeKinds.mpijob, {}).get(
-            mpijob_crd_version
-        ):
-            runtime_handler_instances_cache[RuntimeKinds.mpijob][
-                mpijob_crd_version
-            ] = runtime_handler_class()
-        return runtime_handler_instances_cache[RuntimeKinds.mpijob][mpijob_crd_version]
-
-    kind_runtime_handler_map = {
-        RuntimeKinds.dask: DaskRuntimeHandler,
-        RuntimeKinds.spark: SparkRuntimeHandler,
-        RuntimeKinds.remotespark: RemoteSparkRuntimeHandler,
-        RuntimeKinds.job: KubeRuntimeHandler,
-    }
-    runtime_handler_class = kind_runtime_handler_map[kind]
-    if not runtime_handler_instances_cache.get(kind):
-        runtime_handler_instances_cache[kind] = runtime_handler_class()
-    return runtime_handler_instances_cache[kind]
-
 
 def get_runtime_class(kind: str):
     if kind == RuntimeKinds.mpijob: