mlrun 1.7.1rc10__py3-none-any.whl → 1.8.0rc8__py3-none-any.whl

mlrun/package/packagers_manager.py

@@ -17,10 +17,11 @@ import inspect
 import os
 import shutil
 import traceback
-from typing import Any, Union
+from typing import Any, Optional, Union
 
+import mlrun.errors
 from mlrun.artifacts import Artifact
-from mlrun.datastore import DataItem, store_manager
+from mlrun.datastore import DataItem, get_store_resource, store_manager
 from mlrun.errors import MLRunInvalidArgumentError
 from mlrun.utils import logger
 
@@ -41,7 +42,7 @@ class PackagersManager:
     It prepares the instructions / log hint configurations and then looks for the first packager that fits the task.
     """
 
-    def __init__(self, default_packager: type[Packager] = None):
+    def __init__(self, default_packager: Optional[type[Packager]] = None):
         """
         Initialize a packagers manager.
 
@@ -302,18 +303,17 @@ class PackagersManager:
 
     def link_packages(
         self,
-        additional_artifacts: list[Artifact],
+        additional_artifact_uris: dict,
         additional_results: dict,
     ):
         """
         Link packages to each other according to the provided extra data and metrics spec keys. A future link is
         marked with ellipses (...). If no link is found, None is used and a warning is printed.
 
-        :param additional_artifacts: Additional artifacts to link (should come from an `mlrun.MLClientCtx`).
-        :param additional_results:   Additional results to link (should come from an `mlrun.MLClientCtx`).
+        :param additional_artifact_uris:    Additional artifact URIs to link (should come from an `mlrun.MLClientCtx`).
+        :param additional_results:          Additional results to link (should come from an `mlrun.MLClientCtx`).
         """
         # Join the manager's artifacts and results with the additional ones to look for a link in all of them:
-        joined_artifacts = [*additional_artifacts, *self.artifacts]
         joined_results = {**additional_results, **self.results}
 
         # Go over the artifacts and link:
@@ -324,7 +324,10 @@ class PackagersManager:
                 if artifact.spec.extra_data[key] is ...:
                     # Look for an artifact or result with this key to link it:
                     extra_data = self._look_for_extra_data(
-                        key=key, artifacts=joined_artifacts, results=joined_results
+                        key=key,
+                        artifacts=self.artifacts,
+                        artifact_uris=additional_artifact_uris,
+                        results=joined_results,
                     )
                     # Print a warning if a link is missing:
                     if extra_data is None:
@@ -398,8 +401,8 @@ class PackagersManager:
     def _get_packager_for_packing(
         self,
         obj: Any,
-        artifact_type: str = None,
-        configurations: dict = None,
+        artifact_type: Optional[str] = None,
+        configurations: Optional[dict] = None,
     ) -> Union[Packager, None]:
         """
         Look for a packager that can pack the provided object as the provided artifact type.
@@ -426,7 +429,7 @@ class PackagersManager:
         self,
         data_item: Any,
         type_hint: type,
-        artifact_type: str = None,
+        artifact_type: Optional[str] = None,
     ) -> Union[Packager, None]:
         """
         Look for a packager that can unpack the data item of the given type hint as the provided artifact type.
@@ -715,17 +718,31 @@ class PackagersManager:
     def _look_for_extra_data(
         key: str,
         artifacts: list[Artifact],
+        artifact_uris: dict,
         results: dict,
     ) -> Union[Artifact, str, int, float, None]:
         """
         Look for an extra data item (artifact or result) by given key. If not found, None is returned.
 
-        :param key:       Key to look for.
-        :param artifacts: Artifacts to look in.
-        :param results:   Results to look in.
+        :param key:             Key to look for.
+        :param artifacts:       Artifacts to look in.
+        :param artifact_uris:   Artifacts URIs to look in.
+        :param results:         Results to look in.
 
         :return: The artifact or result with the same key or None if not found.
         """
+        artifact_uris = artifact_uris or {}
+        for _key, uri in artifact_uris.items():
+            if key == _key:
+                try:
+                    return get_store_resource(uri)
+                except mlrun.errors.MLRunNotFoundError as exc:
+                    logger.warn(
+                        f"Artifact {key=} not found when looking for extra data",
+                        exc=mlrun.errors.err_to_str(exc),
+                    )
+                    return None
+
         # Look in the artifacts:
         for artifact in artifacts:
             if key == artifact.key:

mlrun/package/utils/__init__.py

@@ -11,9 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
 from ._archiver import ArchiveSupportedFormat
 from ._formatter import StructFileSupportedFormat

mlrun/package/utils/_pickler.py

@@ -19,7 +19,7 @@ import sys
 import tempfile
 import warnings
 from types import ModuleType
-from typing import Any, Union
+from typing import Any, Optional, Union
 
 from mlrun.errors import MLRunInvalidArgumentError
 from mlrun.utils import logger
@@ -34,7 +34,7 @@ class Pickler:
 
     @staticmethod
     def pickle(
-        obj: Any, pickle_module_name: str, output_path: str = None
+        obj: Any, pickle_module_name: str, output_path: Optional[str] = None
     ) -> tuple[str, dict[str, Union[str, None]]]:
         """
         Pickle an object using the given module. The pickled object will be saved to file to the given output path.
@@ -91,10 +91,10 @@ class Pickler:
     def unpickle(
         pickle_path: str,
         pickle_module_name: str,
-        object_module_name: str = None,
-        python_version: str = None,
-        pickle_module_version: str = None,
-        object_module_version: str = None,
+        object_module_name: Optional[str] = None,
+        python_version: Optional[str] = None,
+        pickle_module_version: Optional[str] = None,
+        object_module_version: Optional[str] = None,
     ) -> Any:
         """
         Unpickle an object using the given instructions. Warnings may be raised in case any of the versions are

mlrun/platforms/__init__.py

@@ -12,23 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 import json
+import warnings
 from pprint import pprint
 from time import sleep
-
-from mlrun_pipelines.common.mounts import VolumeMount
-from mlrun_pipelines.mounts import (
-    auto_mount,
-    mount_configmap,
-    mount_hostpath,
-    mount_pvc,
-    mount_s3,
-    mount_secret,
-    mount_v3io,
-    set_env_variables,
-    v3io_cred,
-)
+from typing import Optional
 
 from .iguazio import (
     V3ioStreamClient,
@@ -37,10 +25,53 @@ from .iguazio import (
 )
 
 
+class _DeprecationHelper:
+    """A helper class to deprecate old schemas"""
+
+    def __init__(self, new_target: str, version="1.8.0"):
+        self._new_target = new_target
+        self._version = version
+
+    def __call__(self, *args, **kwargs):
+        self._warn()
+        return self._lazy_load()(*args, **kwargs)
+
+    def __getattr__(self, attr):
+        self._warn()
+        return getattr(self._lazy_load(), attr)
+
+    def _lazy_load(self, *args, **kwargs):
+        import mlrun.runtimes.mounts as mlrun_mounts
+
+        return getattr(mlrun_mounts, self._new_target)
+
+    def _warn(self):
+        warnings.warn(
+            f"mlrun.platforms.{self._new_target} is deprecated since version {self._version}, "
+            f"and will be removed in 1.10. Use mlrun.runtimes.mounts.{self._new_target} instead.",
+            FutureWarning,
+        )
+
+
+# TODO: Remove in 1.10
+# For backwards compatibility
+VolumeMount = _DeprecationHelper("VolumeMount")
+auto_mount = _DeprecationHelper("auto_mount")
+mount_configmap = _DeprecationHelper("mount_configmap")
+mount_hostpath = _DeprecationHelper("mount_hostpath")
+mount_pvc = _DeprecationHelper("mount_pvc")
+mount_s3 = _DeprecationHelper("mount_s3")
+mount_secret = _DeprecationHelper("mount_secret")
+mount_v3io = _DeprecationHelper("mount_v3io")
+set_env_variables = _DeprecationHelper("set_env_variables")
+v3io_cred = _DeprecationHelper("v3io_cred")
+# eof 'For backwards compatibility'
+
+
 def watch_stream(
     url,
-    shard_ids: list = None,
-    seek_to: str = None,
+    shard_ids: Optional[list] = None,
+    seek_to: Optional[str] = None,
     interval=None,
     is_json=False,
     **kwargs,

mlrun/platforms/iguazio.py

@@ -15,6 +15,7 @@
 import json
 import os
 import urllib
+from typing import Optional
 from urllib.parse import urlparse
 
 import requests
@@ -250,7 +251,9 @@ class KafkaOutputStream:
 
 
 class V3ioStreamClient:
-    def __init__(self, url: str, shard_id: int = 0, seek_to: str = None, **kwargs):
+    def __init__(
+        self, url: str, shard_id: int = 0, seek_to: Optional[str] = None, **kwargs
+    ):
         endpoint, stream_path = parse_path(url)
         seek_options = ["EARLIEST", "LATEST", "TIME", "SEQUENCE"]
         seek_to = seek_to or "LATEST"

mlrun/projects/operations.py

@@ -12,16 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import typing
 import warnings
 from typing import Optional, Union
 
-import mlrun_pipelines.common.models
-import mlrun_pipelines.models
-
 import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas.function
 import mlrun.common.schemas.workflow
+import mlrun_pipelines.common.models
+import mlrun_pipelines.models
 from mlrun.utils import hub_prefix
 
 from .pipelines import enrich_function_object, pipeline_context
@@ -59,25 +59,25 @@ def _get_engine_and_function(function, project=None):
 
 def run_function(
     function: Union[str, mlrun.runtimes.BaseRuntime],
-    handler: str = None,
+    handler: Optional[Union[str, typing.Callable]] = None,
     name: str = "",
-    params: dict = None,
-    hyperparams: dict = None,
+    params: Optional[dict] = None,
+    hyperparams: Optional[dict] = None,
     hyper_param_options: mlrun.model.HyperParamOptions = None,
-    inputs: dict = None,
-    outputs: list[str] = None,
+    inputs: Optional[dict] = None,
+    outputs: Optional[list[str]] = None,
     workdir: str = "",
-    labels: dict = None,
+    labels: Optional[dict] = None,
     base_task: mlrun.model.RunTemplate = None,
     watch: bool = True,
-    local: bool = None,
-    verbose: bool = None,
-    selector: str = None,
+    local: Optional[bool] = None,
+    verbose: Optional[bool] = None,
+    selector: Optional[str] = None,
     project_object=None,
-    auto_build: bool = None,
+    auto_build: Optional[bool] = None,
     schedule: Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
-    artifact_path: str = None,
-    notifications: list[mlrun.model.Notification] = None,
+    artifact_path: Optional[str] = None,
+    notifications: Optional[list[mlrun.model.Notification]] = None,
     returns: Optional[list[Union[str, dict[str, str]]]] = None,
     builder_env: Optional[list] = None,
     reset_on_run: Optional[bool] = None,
@@ -255,19 +255,19 @@ class BuildStatus:
 
 def build_function(
     function: Union[str, mlrun.runtimes.BaseRuntime],
-    with_mlrun: bool = None,
+    with_mlrun: Optional[bool] = None,
     skip_deployed: bool = False,
     image=None,
     base_image=None,
-    commands: list = None,
+    commands: Optional[list] = None,
     secret_name=None,
-    requirements: Union[str, list[str]] = None,
-    requirements_file: str = None,
+    requirements: Optional[Union[str, list[str]]] = None,
+    requirements_file: Optional[str] = None,
     mlrun_version_specifier=None,
-    builder_env: dict = None,
+    builder_env: Optional[dict] = None,
     project_object=None,
     overwrite_build_params: bool = False,
-    extra_args: str = None,
+    extra_args: Optional[str] = None,
     force_build: bool = False,
 ) -> Union[BuildStatus, mlrun_pipelines.models.PipelineNodeWrapper]:
     """deploy ML function, build container with its dependencies
@@ -371,13 +371,13 @@ class DeployStatus:
 
 def deploy_function(
     function: Union[str, mlrun.runtimes.BaseRuntime],
-    models: list = None,
-    env: dict = None,
-    tag: str = None,
-    verbose: bool = None,
-    builder_env: dict = None,
+    models: Optional[list] = None,
+    env: Optional[dict] = None,
+    tag: Optional[str] = None,
+    verbose: Optional[bool] = None,
+    builder_env: Optional[dict] = None,
     project_object=None,
-    mock: bool = None,
+    mock: Optional[bool] = None,
 ) -> Union[DeployStatus, mlrun_pipelines.models.PipelineNodeWrapper]:
     """deploy real-time (nuclio based) functions
 

mlrun/projects/pipelines.py

@@ -20,16 +20,15 @@ import tempfile
 import typing
 import uuid
 
-import mlrun_pipelines.common.models
-import mlrun_pipelines.patcher
-import mlrun_pipelines.utils
-
 import mlrun
 import mlrun.common.runtimes.constants
 import mlrun.common.schemas
 import mlrun.common.schemas.function
 import mlrun.common.schemas.workflow
 import mlrun.utils.notifications
+import mlrun_pipelines.common.models
+import mlrun_pipelines.patcher
+import mlrun_pipelines.utils
 from mlrun.errors import err_to_str
 from mlrun.utils import (
     get_ui_url,
@@ -40,7 +39,7 @@ from mlrun.utils import (
 
 from ..common.helpers import parse_versioned_object_uri
 from ..config import config
-from ..run import _run_pipeline, wait_for_pipeline_completion
+from ..run import _run_pipeline, retry_pipeline, wait_for_pipeline_completion
 from ..runtimes.pod import AutoMountType
 
 
@@ -317,7 +316,7 @@ def get_db_function(project, key) -> mlrun.runtimes.BaseRuntime:
 def enrich_function_object(
     project: mlrun.common.schemas.Project,
     function: mlrun.runtimes.BaseRuntime,
-    decorator: typing.Callable = None,
+    decorator: typing.Optional[typing.Callable] = None,
     copy_function: bool = True,
     try_auto_mount: bool = True,
 ) -> mlrun.runtimes.BaseRuntime:
@@ -380,7 +379,7 @@ class _PipelineRunStatus:
         project: "mlrun.projects.MlrunProject",
         workflow: WorkflowSpec = None,
         state: mlrun_pipelines.common.models.RunStatuses = "",
-        exc: Exception = None,
+        exc: typing.Optional[Exception] = None,
     ):
         """
         :param run_id:      unique id of the pipeline run
@@ -422,6 +421,13 @@ class _PipelineRunStatus:
             self._state = returned_state
         return self._state
 
+    def retry(self) -> str:
+        run_id = self._engine.retry(
+            self,
+            project=self.project,
+        )
+        return run_id
+
     def __str__(self):
         return str(self.run_id)
 
@@ -441,6 +447,17 @@ class _PipelineRunner(abc.ABC):
             f"Save operation not supported in {cls.engine} pipeline engine"
         )
 
+    @classmethod
+    @abc.abstractmethod
+    def retry(
+        cls,
+        run: "_PipelineRunStatus",
+        project: typing.Optional["mlrun.projects.MlrunProject"] = None,
+    ) -> str:
+        raise NotImplementedError(
+            f"Retry operation not supported in {cls.engine} pipeline engine"
+        )
+
     @classmethod
     @abc.abstractmethod
     def run(
@@ -453,7 +470,7 @@ class _PipelineRunner(abc.ABC):
         artifact_path=None,
         namespace=None,
         source=None,
-        notifications: list[mlrun.model.Notification] = None,
+        notifications: typing.Optional[list[mlrun.model.Notification]] = None,
     ) -> _PipelineRunStatus:
         pass
 
@@ -463,7 +480,7 @@ class _PipelineRunner(abc.ABC):
         run: "_PipelineRunStatus",
         project: typing.Optional["mlrun.projects.MlrunProject"] = None,
         timeout: typing.Optional[int] = None,
-        expected_statuses: list[str] = None,
+        expected_statuses: typing.Optional[list[str]] = None,
     ):
         pass
 
@@ -577,7 +594,7 @@ class _KFPRunner(_PipelineRunner):
         artifact_path=None,
         namespace=None,
         source=None,
-        notifications: list[mlrun.model.Notification] = None,
+        notifications: typing.Optional[list[mlrun.model.Notification]] = None,
     ) -> _PipelineRunStatus:
         pipeline_context.set(project, workflow_spec)
         workflow_handler = _PipelineRunner._get_handler(
@@ -636,12 +653,30 @@ class _KFPRunner(_PipelineRunner):
         pipeline_context.clear()
         return _PipelineRunStatus(run_id, cls, project=project, workflow=workflow_spec)
 
+    @classmethod
+    def retry(
+        cls,
+        run: "_PipelineRunStatus",
+        project: typing.Optional["mlrun.projects.MlrunProject"] = None,
+    ) -> str:
+        project_name = project.metadata.name if project else ""
+        logger.info(
+            "Retrying pipeline",
+            run_id=run.run_id,
+            project=project_name,
+        )
+        run_id = retry_pipeline(
+            run.run_id,
+            project=project_name,
+        )
+        return run_id
+
     @staticmethod
     def wait_for_completion(
         run: "_PipelineRunStatus",
         project: typing.Optional["mlrun.projects.MlrunProject"] = None,
         timeout: typing.Optional[int] = None,
-        expected_statuses: list[str] = None,
+        expected_statuses: typing.Optional[list[str]] = None,
     ):
         project_name = project.metadata.name if project else ""
         logger.info(
@@ -686,7 +721,7 @@ class _LocalRunner(_PipelineRunner):
         artifact_path=None,
         namespace=None,
         source=None,
-        notifications: list[mlrun.model.Notification] = None,
+        notifications: typing.Optional[list[mlrun.model.Notification]] = None,
     ) -> _PipelineRunStatus:
         pipeline_context.set(project, workflow_spec)
         workflow_handler = _PipelineRunner._get_handler(
@@ -763,13 +798,13 @@ class _RemoteRunner(_PipelineRunner):
         cls,
         project: "mlrun.projects.MlrunProject",
         workflow_spec: WorkflowSpec,
-        name: str = None,
-        workflow_handler: typing.Union[str, typing.Callable] = None,
+        name: typing.Optional[str] = None,
+        workflow_handler: typing.Optional[typing.Union[str, typing.Callable]] = None,
         secrets: mlrun.secrets.SecretsStore = None,
-        artifact_path: str = None,
-        namespace: str = None,
-        source: str = None,
-        notifications: list[mlrun.model.Notification] = None,
+        artifact_path: typing.Optional[str] = None,
+        namespace: typing.Optional[str] = None,
+        source: typing.Optional[str] = None,
+        notifications: typing.Optional[list[mlrun.model.Notification]] = None,
     ) -> typing.Optional[_PipelineRunStatus]:
         workflow_name = normalize_workflow_name(name=name, project_name=project.name)
         workflow_id = None
@@ -890,7 +925,7 @@ class _RemoteRunner(_PipelineRunner):
         timeout=None,
         expected_statuses=None,
         notifiers: mlrun.utils.notifications.CustomNotificationPusher = None,
-        inner_engine: type[_PipelineRunner] = None,
+        inner_engine: typing.Optional[type[_PipelineRunner]] = None,
     ):
         inner_engine = inner_engine or _KFPRunner
         if inner_engine.engine == _KFPRunner.engine:
@@ -998,25 +1033,25 @@ def load_and_run(context, *args, **kwargs):
 
 def load_and_run_workflow(
     context: mlrun.execution.MLClientCtx,
-    url: str = None,
+    url: typing.Optional[str] = None,
     project_name: str = "",
-    init_git: bool = None,
-    subpath: str = None,
+    init_git: typing.Optional[bool] = None,
+    subpath: typing.Optional[str] = None,
     clone: bool = False,
-    workflow_name: str = None,
-    workflow_path: str = None,
-    workflow_arguments: dict[str, typing.Any] = None,
-    artifact_path: str = None,
-    workflow_handler: typing.Union[str, typing.Callable] = None,
-    namespace: str = None,
+    workflow_name: typing.Optional[str] = None,
+    workflow_path: typing.Optional[str] = None,
+    workflow_arguments: typing.Optional[dict[str, typing.Any]] = None,
+    artifact_path: typing.Optional[str] = None,
+    workflow_handler: typing.Optional[typing.Union[str, typing.Callable]] = None,
+    namespace: typing.Optional[str] = None,
     sync: bool = False,
     dirty: bool = False,
-    engine: str = None,
-    local: bool = None,
+    engine: typing.Optional[str] = None,
+    local: typing.Optional[bool] = None,
     schedule: typing.Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
-    cleanup_ttl: int = None,
+    cleanup_ttl: typing.Optional[int] = None,
     wait_for_completion: bool = False,
-    project_context: str = None,
+    project_context: typing.Optional[str] = None,
 ):
     """
     Auxiliary function that the RemoteRunner run once or run every schedule.
@@ -1220,13 +1255,13 @@ def handle_workflow_completion(
 
 def import_remote_project(
     context: mlrun.execution.MLClientCtx,
-    url: str = None,
+    url: typing.Optional[str] = None,
     project_name: str = "",
-    init_git: bool = None,
-    subpath: str = None,
+    init_git: typing.Optional[bool] = None,
+    subpath: typing.Optional[str] = None,
     clone: bool = False,
     save: bool = True,
-    project_context: str = None,
+    project_context: typing.Optional[str] = None,
 ):
     """
     This function loads a project from a given remote source.