mlrun 1.5.0rc1__py3-none-any.whl → 1.5.0rc2__py3-none-any.whl

mlrun/package/packager.py

@@ -107,8 +107,7 @@ class Packager(ABC, metaclass=_PackagerMeta):
 
     Preferably, each packager should handle a single type of object.
 
-    Linking Artifacts (extra data)
-    ------------------------------
+    **Linking Artifacts (extra data)**
 
     In order to link between packages (using the extra data or metrics spec attributes of an artifact), you should use
     the key as if it exists and as value ellipses (...). The manager will link all packages once it is done packing.
@@ -118,8 +117,7 @@ class Packager(ABC, metaclass=_PackagerMeta):
         artifact = Artifact(key="my_artifact")
         artifact.spec.extra_data = {key: ... for key in extra_data}
 
-    Clearing Outputs
-    ----------------
+    **Clearing Outputs**
 
     Some of the packagers may produce files and temporary directories that should be deleted once done with logging the
     artifact. The packager can mark paths of files and directories to delete after logging using the class method
@@ -131,15 +129,15 @@ class Packager(ABC, metaclass=_PackagerMeta):
         with open("./some_file.txt", "w") as file:
             file.write("Pack me")
         artifact = Artifact(key="my_artifact")
-        cls.future_clear(path="./some_file.txt")
+        cls.add_future_clearing_path(path="./some_file.txt")
         return artifact, None
     """
 
-    # The type of object this packager can pack and unpack:
+    #: The type of object this packager can pack and unpack.
     PACKABLE_OBJECT_TYPE: Type = ...
 
-    # The priority of this packager in the packagers collection of the manager (lower is better)
-    PRIORITY = ...
+    #: The priority of this packager in the packagers collection of the manager (lower is better).
+    PRIORITY: int = ...
 
     # List of all paths to be deleted by the manager of this packager post logging the packages:
     _CLEARING_PATH_LIST: List[str] = []

mlrun/package/packagers/default_packager.py

@@ -16,16 +16,123 @@ import inspect
 from types import MethodType
 from typing import Any, List, Tuple, Type, Union
 
+import docstring_parser
+
 from mlrun.artifacts import Artifact
 from mlrun.datastore import DataItem
 from mlrun.utils import logger
 
 from ..errors import MLRunPackagePackingError, MLRunPackageUnpackingError
-from ..packager import Packager
+from ..packager import Packager, _PackagerMeta
 from ..utils import DEFAULT_PICKLE_MODULE, ArtifactType, Pickler, TypeHintUtils
 
 
-class DefaultPackager(Packager):
+class _DefaultPackagerMeta(_PackagerMeta):
+    """
+    Metaclass for `DefaultPackager` to override `__doc__` attribute into a class property. This way sphinx will get a
+    dynamically generated docstring that will include a summary of the packager.
+    """
+
+    def __new__(mcls, name: str, bases: tuple, namespace: dict, **kwargs):
+        """
+        Create a new DefaultPackager metaclass that saves the original packager docstring to another attribute named
+        `_packager_doc`.
+
+        :param name:      A string representing the name of the class being instantiated.
+        :param bases:     A tuple of classes from which the class will inherit.
+        :param namespace: The namespace of the class holding its attributes (from here the docstring will be taken).
+        """
+        # Save the original doc string to a separate class variable as it will be overriden later on by the metaclass
+        # property `__doc__`:
+        namespace["_packager_doc"] = namespace.get("__doc__", "")
+
+        # Continue creating the metaclass:
+        return super().__new__(mcls, name, bases, namespace, **kwargs)
+
+    @property
+    def __doc__(cls) -> str:
+        """
+        Override the `__doc__` attribute of a `DefaultPackager` to be a property in order to auto-summarize the
+        packager's class docstring. The summary is concatenated after the original class doc string.
+
+        The summary will be in the following structure:
+
+        <cls._packager_doc>
+
+        .. rubric:: Packager Summary
+
+        **Packing Type**: ``<cls.PACKABLE_OBJECT_TYPE>``
+
+        **Packing Sub-Classes**: True / False
+
+        **Artifact Types**:
+
+        * **type 1**: ...
+
+          * configuration 1 - ...
+          * configuration 2 - ...
+
+        * **type 2**: ...
+
+          * configuration 1: ...
+          * configuration 2: ...
+
+        :returns: The original docstring with the generated packager summary.
+        """
+        # Get the original packager class doc string:
+        packager_doc_string = cls._packager_doc.split("\n")
+        packager_doc_string = "\n".join(line[4:] for line in packager_doc_string)
+
+        # Parse the packable type section:
+        type_name = (
+            "Any type"
+            if cls.PACKABLE_OBJECT_TYPE is ...
+            else (
+                f"``{str(cls.PACKABLE_OBJECT_TYPE)}``"
+                if TypeHintUtils.is_typing_type(type_hint=cls.PACKABLE_OBJECT_TYPE)
+                else f"``{cls.PACKABLE_OBJECT_TYPE.__module__}.{cls.PACKABLE_OBJECT_TYPE.__name__}``"
+            )
+        )
+        packing_type = f"**Packing Type**: {type_name}"
+
+        # Subclasses support section:
+        packing_sub_classes = f"**Packing Sub-Classes**: {cls.PACK_SUBCLASSES}"
+
+        # Artifact types section:
+        artifact_types = "**Artifact Types**:"
+        for artifact_type in cls.get_supported_artifact_types():
+            # Get the packing method docstring:
+            method_doc = docstring_parser.parse(
+                getattr(cls, f"pack_{artifact_type}").__doc__
+            )
+            # Add the artifact type bullet:
+            artifact_type_doc = f"{method_doc.short_description or ''}{method_doc.long_description or ''}".replace(
+                "\n", ""
+            )
+            artifact_types += f"\n\n* **{artifact_type}** - " + artifact_type_doc
+            # Add the artifact type configurations (ignoring the `obj` and `key` parameters):
+            configurations_doc = "\n\n  * ".join(
+                "{} - {}".format(
+                    parameter.arg_name, parameter.description.replace("\n", "")
+                )
+                for parameter in method_doc.params[2:]
+            )
+            if configurations_doc:
+                artifact_types += f"\n\n  * {configurations_doc}"
+
+        # Construct the final doc string and return:
+        doc = (
+            f"{packager_doc_string}"
+            "\n\n.. rubric:: Packager Summary"
+            f"\n\n{packing_type}"
+            f"\n\n{packing_sub_classes}"
+            f"\n\n{artifact_types}"
+            f"\n\n"
+        )
+        return doc
+
+
+class DefaultPackager(Packager, metaclass=_DefaultPackagerMeta):
     """
     A default packager that handles all types and pack them as pickle files.
 
@@ -89,7 +196,7 @@ class DefaultPackager(Packager):
 
     Important to remember (from the ``Packager`` docstring):
 
-    * Linking artifacts ("extra data"): In order to link between packages (using the extra data or metrics spec
+    * **Linking artifacts** ("extra data"): In order to link between packages (using the extra data or metrics spec
       attributes of an artifact), you should use the key as if it exists and as value ellipses (...). The manager will
       link all packages once it is done packing.
 
@@ -98,9 +205,9 @@ class DefaultPackager(Packager):
           artifact = Artifact(key="my_artifact")
           artifact.spec.extra_data = {key: ... for key in extra_data}
 
-    * Clearing outputs: Some packagers may produce files and temporary directories that should be deleted once done with
-      logging the artifact. The packager can mark paths of files and directories to delete after logging using the class
-      method ``future_clear``.
+    * **Clearing outputs**: Some packagers may produce files and temporary directories that should be deleted once done
+      with logging the artifact. The packager can mark paths of files and directories to delete after logging using the
+      class method ``future_clear``.
 
       For example, in the following packager's ``pack`` method we can write a text file, create an Artifact and then
       mark the text file to be deleted once the artifact is logged::
@@ -110,15 +217,19 @@ class DefaultPackager(Packager):
           artifact = Artifact(key="my_artifact")
           cls.future_clear(path="./some_file.txt")
           return artifact, None
+
     """
 
-    # The type of object this packager can pack and unpack:
+    #: The type of object this packager can pack and unpack.
     PACKABLE_OBJECT_TYPE: Type = ...
-    # A flag for indicating whether to pack all subclasses of the `PACKABLE_OBJECT_TYPE` as well:
+
+    #: A flag for indicating whether to pack all subclasses of the `PACKABLE_OBJECT_TYPE` as well.
     PACK_SUBCLASSES = False
-    # The default artifact type to pack as:
+
+    #: The default artifact type to pack as.
     DEFAULT_PACKING_ARTIFACT_TYPE = ArtifactType.OBJECT
-    # The default artifact type to unpack from:
+
+    #: The default artifact type to unpack from.
     DEFAULT_UNPACKING_ARTIFACT_TYPE = ArtifactType.OBJECT
 
     @classmethod

mlrun/platforms/__init__.py

@@ -23,8 +23,6 @@ from .iguazio import (
     add_or_refresh_credentials,
     is_iguazio_session_cookie,
     mount_v3io,
-    mount_v3io_extended,
-    mount_v3io_legacy,
     v3io_cred,
 )
 from .other import (

mlrun/platforms/iguazio.py

@@ -25,7 +25,6 @@ import requests
 import semver
 import urllib3
 import v3io
-from deprecated import deprecated
 
 import mlrun.errors
 from mlrun.config import config as mlconf
@@ -37,34 +36,6 @@ _cached_control_session = None
 VolumeMount = namedtuple("Mount", ["path", "sub_path"])
 
 
-# TODO: Remove in 1.5.0
-@deprecated(
-    version="1.3.0",
-    reason="'mount_v3io_extended' will be removed in 1.5.0, use 'mount_v3io' instead",
-    category=FutureWarning,
-)
-def mount_v3io_extended(
-    name="v3io", remote="", mounts=None, access_key="", user="", secret=None
-):
-    """Modifier function to apply to a Container Op to volume mount a v3io path
-    :param name:            the volume name
-    :param remote:          the v3io path to use for the volume. ~/ prefix will be replaced with /users/<username>/
-    :param mounts:          list of mount & volume sub paths (type VolumeMount).
-                            empty mounts & remote will default to mount /v3io & /User.
-    :param access_key:      the access key used to auth against v3io. if not given V3IO_ACCESS_KEY env var will be used
-    :param user:            the username used to auth against v3io. if not given V3IO_USERNAME env var will be used
-    :param secret:          k8s secret name which would be used to get the username and access key to auth against v3io.
-    """
-    return mount_v3io(
-        name=name,
-        remote=remote,
-        volume_mounts=mounts,
-        access_key=access_key,
-        user=user,
-        secret=secret,
-    )
-
-
 def mount_v3io(
     name="v3io",
     remote="",
@@ -109,33 +80,6 @@ def mount_v3io(
     return _attach_volume_mounts_and_creds
 
 
-# TODO: Remove in 1.5.0
-@deprecated(
-    version="1.3.0",
-    reason="'mount_v3io_legacy' will be removed in 1.5.0, use 'mount_v3io' instead",
-    category=FutureWarning,
-)
-def mount_v3io_legacy(
-    name="v3io", remote="~/", mount_path="/User", access_key="", user="", secret=None
-):
-    """Modifier function to apply to a Container Op to volume mount a v3io path
-    :param name:            the volume name
-    :param remote:          the v3io path to use for the volume. ~/ prefix will be replaced with /users/<username>/
-    :param mount_path:      the volume mount path
-    :param access_key:      the access key used to auth against v3io. if not given V3IO_ACCESS_KEY env var will be used
-    :param user:            the username used to auth against v3io. if not given V3IO_USERNAME env var will be used
-    :param secret:          k8s secret name which would be used to get the username and access key to auth against v3io.
-    """
-    return mount_v3io(
-        name=name,
-        remote=remote,
-        volume_mounts=[VolumeMount(path=mount_path, sub_path="")],
-        access_key=access_key,
-        user=user,
-        secret=secret,
-    )
-
-
 def _enrich_and_validate_v3io_mounts(remote="", volume_mounts=None, user=""):
     if remote and not volume_mounts:
         raise mlrun.errors.MLRunInvalidArgumentError(

mlrun/projects/pipelines.py

@@ -16,11 +16,9 @@ import builtins
 import importlib.util as imputil
 import os
 import tempfile
-import time
 import traceback
 import typing
 import uuid
-import warnings
 
 import kfp.compiler
 from kfp import dsl
@@ -30,7 +28,13 @@ import mlrun
 import mlrun.common.schemas
 import mlrun.utils.notifications
 from mlrun.errors import err_to_str
-from mlrun.utils import get_ui_url, logger, new_pipe_metadata
+from mlrun.utils import (
+    get_ui_url,
+    logger,
+    new_pipe_metadata,
+    normalize_workflow_name,
+    retry_until_successful,
+)
 
 from ..common.helpers import parse_versioned_object_uri
 from ..config import config
@@ -72,32 +76,23 @@ class WorkflowSpec(mlrun.model.ModelObj):
         args=None,
         name=None,
         handler=None,
-        # TODO: deprecated, remove in 1.5.0
-        ttl=None,
         args_schema: dict = None,
         schedule: typing.Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
         cleanup_ttl: int = None,
+        image: str = None,
     ):
-        if ttl:
-            warnings.warn(
-                "'ttl' is deprecated, use 'cleanup_ttl' instead. "
-                "This will be removed in 1.5.0",
-                # TODO: Remove this in 1.5.0
-                FutureWarning,
-            )
-
         self.engine = engine
         self.code = code
         self.path = path
         self.args = args
         self.name = name
         self.handler = handler
-        self.ttl = cleanup_ttl or ttl
-        self.cleanup_ttl = cleanup_ttl or ttl
+        self.cleanup_ttl = cleanup_ttl
         self.args_schema = args_schema
         self.run_local = False
         self._tmp_path = None
         self.schedule = schedule
+        self.image = image
 
     def get_source_file(self, context=""):
         if not self.code and not self.path:
@@ -553,7 +548,7 @@ class _KFPRunner(_PipelineRunner):
 
         conf = new_pipe_metadata(
             artifact_path=artifact_path,
-            cleanup_ttl=workflow_spec.cleanup_ttl or workflow_spec.ttl,
+            cleanup_ttl=workflow_spec.cleanup_ttl,
         )
         compiler.Compiler().compile(pipeline, target, pipeline_conf=conf)
         workflow_spec.clear_tmp()
@@ -586,7 +581,7 @@ class _KFPRunner(_PipelineRunner):
             experiment=name or workflow_spec.name,
             namespace=namespace,
             artifact_path=artifact_path,
-            cleanup_ttl=workflow_spec.cleanup_ttl or workflow_spec.ttl,
+            cleanup_ttl=workflow_spec.cleanup_ttl,
         )
         project.notifiers.push_pipeline_start_message(
             project.metadata.name,
@@ -747,137 +742,56 @@ class _RemoteRunner(_PipelineRunner):
 
     engine = "remote"
 
-    @staticmethod
-    def _prepare_load_and_run_function(
-        source: str,
-        project_name: str,
-        save: bool,
-        workflow_name: str,
-        workflow_spec: WorkflowSpec,
-        artifact_path: str,
-        workflow_handler: str,
-        namespace: str,
-        subpath: str,
-    ) -> typing.Tuple[mlrun.runtimes.RemoteRuntime, "mlrun.RunObject"]:
-        """
-        Helper function for creating the runspec of the load and run function.
-        For internal use only.
-        :param source:              The source of the project.
-        :param project_name:        project name
-        :param save:                either to save the project in the DB
-        :param workflow_name:       workflow name
-        :param workflow_spec:       workflow to run
-        :param artifact_path:       path to store artifacts
-        :param workflow_handler:    workflow function handler (for running workflow function directly)
-        :param namespace:           kubernetes namespace if other than default
-        :param subpath:             project subpath (within the archive)
-        :return:
-        """
-        # Creating the load project and workflow running function:
-        load_and_run_fn = mlrun.new_function(
-            name=mlrun.mlconf.default_workflow_runner_name.format(workflow_name),
-            project=project_name,
-            kind="job",
-            image=mlrun.mlconf.default_base_image,
-        )
-        runspec = mlrun.RunObject(
-            spec=mlrun.model.RunSpec(
-                parameters={
-                    "url": source,
-                    "project_name": project_name,
-                    "save": save,
-                    "workflow_name": workflow_name or workflow_spec.name,
-                    "workflow_path": workflow_spec.path,
-                    "workflow_arguments": workflow_spec.args,
-                    "artifact_path": artifact_path,
-                    "workflow_handler": workflow_handler or workflow_spec.handler,
-                    "namespace": namespace,
-                    "ttl": workflow_spec.cleanup_ttl or workflow_spec.ttl,
-                    "engine": workflow_spec.engine,
-                    "local": workflow_spec.run_local,
-                    "schedule": workflow_spec.schedule,
-                    "subpath": subpath,
-                },
-                handler="mlrun.projects.load_and_run",
-            ),
-            metadata=mlrun.model.RunMetadata(name=workflow_name),
-        )
-
-        runspec = runspec.set_label("job-type", "workflow-runner").set_label(
-            "workflow", workflow_name
-        )
-        return load_and_run_fn, runspec
-
     @classmethod
     def run(
         cls,
-        project,
+        project: "mlrun.projects.MlrunProject",
         workflow_spec: WorkflowSpec,
-        name=None,
-        workflow_handler=None,
-        secrets=None,
-        artifact_path=None,
-        namespace=None,
-        source=None,
+        name: str = None,
+        workflow_handler: typing.Union[str, typing.Callable] = None,
+        secrets: mlrun.secrets.SecretsStore = None,
+        artifact_path: str = None,
+        namespace: str = None,
+        source: str = None,
     ) -> typing.Optional[_PipelineRunStatus]:
-        workflow_name = name.split("-")[-1] if f"{project.name}-" in name else name
-
-        run_id = None
-
-        # If the user provided a source we want to load the project from the source
-        # (like from a specific commit/branch from git repo) without changing the source of the project (save=False).
-        save, current_source = (
-            (False, source) if source else (True, project.spec.source)
-        )
-        if "://" not in current_source:
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                f"Remote workflows can only be performed by a project with remote source (e.g git:// or http://),"
-                f" but the specified source '{current_source}' is not remote. "
-                f"Either put your code in Git, or archive it and then set a source to it."
-                f" For more details, read"
-                f" https://docs.mlrun.org/en/latest/concepts/scheduled-jobs.html#scheduling-a-workflow"
-            )
-
-        # Creating the load project and workflow running function:
-        load_and_run_fn, runspec = cls._prepare_load_and_run_function(
-            source=current_source,
-            project_name=project.name,
-            save=save,
-            workflow_name=workflow_name,
-            workflow_spec=workflow_spec,
-            artifact_path=artifact_path,
-            workflow_handler=workflow_handler,
-            namespace=namespace,
-            subpath=project.spec.subpath,
-        )
+        workflow_name = normalize_workflow_name(name=name, project_name=project.name)
+        workflow_id = None
 
         # The returned engine for this runner is the engine of the workflow.
         # In this way wait_for_completion/get_run_status would be executed by the correct pipeline runner.
         inner_engine = get_workflow_engine(workflow_spec.engine)
-
-        msg = "executing workflow"
-        if workflow_spec.schedule:
-            msg += " scheduling"
-        logger.info(
-            f"{msg} '{load_and_run_fn.metadata.name}' remotely with {workflow_spec.engine} engine"
-        )
-
+        run_db = mlrun.get_run_db()
         try:
-            run = load_and_run_fn.run(
-                runspec=runspec,
-                local=False,
-                schedule=workflow_spec.schedule,
+            workflow_response = run_db.submit_workflow(
+                project=project.name,
+                name=workflow_name,
+                workflow_spec=workflow_spec,
                 artifact_path=artifact_path,
+                source=source,
+                run_name=config.workflows.default_workflow_runner_name.format(
+                    workflow_name
+                ),
+                namespace=namespace,
             )
             if workflow_spec.schedule:
                 return
-            # Fetching workflow id:
-            while not run_id:
-                run.refresh()
-                run_id = run.status.results.get("workflow_id", None)
-                time.sleep(1)
+
+            # Getting workflow id from run:
+            response = retry_until_successful(
+                1,
+                getattr(mlrun.mlconf.workflows.timeouts, inner_engine.engine),
+                logger,
+                False,
+                run_db.get_workflow_id,
+                project=project.name,
+                name=workflow_response.name,
+                run_id=workflow_response.run_id,
+                engine=workflow_spec.engine,
+            )
+            workflow_id = response.workflow_id
             # After fetching the workflow_id the workflow executed successfully
             state = mlrun.run.RunStatuses.succeeded
+            pipeline_context.clear()
 
         except Exception as e:
             trace = traceback.format_exc()
@@ -888,8 +802,8 @@ class _RemoteRunner(_PipelineRunner):
             )
             state = mlrun.run.RunStatuses.failed
             return _PipelineRunStatus(
-                run_id,
-                inner_engine,
+                run_id=workflow_id,
+                engine=inner_engine,
                 project=project,
                 workflow=workflow_spec,
                 state=state,
@@ -900,8 +814,8 @@ class _RemoteRunner(_PipelineRunner):
         )
         pipeline_context.clear()
         return _PipelineRunStatus(
-            run_id,
-            inner_engine,
+            run_id=workflow_id,
+            engine=inner_engine,
             project=project,
             workflow=workflow_spec,
             state=state,
@@ -919,15 +833,6 @@ def create_pipeline(project, pipeline, functions, secrets=None, handler=None):
     setattr(mod, "functions", functions)
     setattr(mod, "this_project", project)
 
-    if hasattr(mod, "init_functions"):
-        # TODO: remove in 1.5.0
-        warnings.warn(
-            "'init_functions' is deprecated in 1.3.0 and will be removed in 1.5.0. "
-            "Place function initialization in the pipeline code.",
-            FutureWarning,
-        )
-        getattr(mod, "init_functions")(functions, project, secrets)
-
     # verify all functions are in this project (init_functions may add new functions)
     for f in functions.values():
         f.metadata.project = project.metadata.name
@@ -976,12 +881,11 @@ def load_and_run(
     namespace: str = None,
     sync: bool = False,
     dirty: bool = False,
-    # TODO: deprecated, remove in 1.5.0
-    ttl: int = None,
     engine: str = None,
     local: bool = None,
     schedule: typing.Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
     cleanup_ttl: int = None,
+    load_only: bool = False,
 ):
     """
     Auxiliary function that the RemoteRunner run once or run every schedule.
@@ -1004,23 +908,14 @@ def load_and_run(
     :param namespace:           kubernetes namespace if other than default
     :param sync:                force functions sync before run
     :param dirty:               allow running the workflow when the git repo is dirty
-    :param ttl:                 pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
-                                workflow and all its resources are deleted) (deprecated, use cleanup_ttl instead)
     :param engine:              workflow engine running the workflow.
                                 supported values are 'kfp' (default) or 'local'
     :param local:               run local pipeline with local functions (set local=True in function.run())
     :param schedule:            ScheduleCronTrigger class instance or a standard crontab expression string
     :param cleanup_ttl:         pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
                                 workflow and all its resources are deleted)
+    :param load_only:           for just loading the project, inner use.
     """
-    if ttl:
-        warnings.warn(
-            "'ttl' is deprecated, use 'cleanup_ttl' instead. "
-            "This will be removed in 1.5.0",
-            # TODO: Remove this in 1.5.0
-            FutureWarning,
-        )
-
     try:
         project = mlrun.load_project(
             context=f"./{project_name}",
@@ -1030,6 +925,7 @@ def load_and_run(
             subpath=subpath,
             clone=clone,
             save=save,
+            sync_functions=True,
         )
     except Exception as error:
         if schedule:
@@ -1056,6 +952,9 @@ def load_and_run(
 
     context.logger.info(f"Loaded project {project.name} from remote successfully")
 
+    if load_only:
+        return
+
     workflow_log_message = workflow_name or workflow_path
     context.logger.info(f"Running workflow {workflow_log_message} from remote")
     run = project.run(
@@ -1068,7 +967,7 @@ def load_and_run(
         sync=sync,
         watch=False,  # Required for fetching the workflow_id
         dirty=dirty,
-        cleanup_ttl=cleanup_ttl or ttl,
+        cleanup_ttl=cleanup_ttl,
         engine=engine,
         local=local,
     )