mlrun 1.7.2rc4__py3-none-any.whl → 1.8.0__py3-none-any.whl

mlrun/package/packagers/python_standard_library_packagers.py

@@ -15,7 +15,7 @@
 import os
 import pathlib
 import tempfile
-from typing import Union
+from typing import Optional, Union
 
 from mlrun.artifacts import Artifact
 from mlrun.datastore import DataItem
@@ -140,7 +140,7 @@ class StrPackager(DefaultPackager):
         self,
         data_item: DataItem,
         is_directory: bool = False,
-        archive_format: str = None,
+        archive_format: Optional[str] = None,
     ) -> str:
         """
         Unpack a data item representing a path string. If the path is of a file, the file is downloaded to a local
@@ -222,7 +222,7 @@ class _BuiltinCollectionPackager(DefaultPackager):
         return artifact, instructions
 
     def unpack_file(
-        self, data_item: DataItem, file_format: str = None
+        self, data_item: DataItem, file_format: Optional[str] = None
     ) -> Union[dict, list]:
         """
         Unpack a builtin collection from file.
@@ -259,7 +259,9 @@ class DictPackager(_BuiltinCollectionPackager):
 
     PACKABLE_OBJECT_TYPE = dict
 
-    def unpack_file(self, data_item: DataItem, file_format: str = None) -> dict:
+    def unpack_file(
+        self, data_item: DataItem, file_format: Optional[str] = None
+    ) -> dict:
         """
         Unpack a dictionary from file.
 
@@ -285,7 +287,9 @@ class ListPackager(_BuiltinCollectionPackager):
 
     PACKABLE_OBJECT_TYPE = list
 
-    def unpack_file(self, data_item: DataItem, file_format: str = None) -> list:
+    def unpack_file(
+        self, data_item: DataItem, file_format: Optional[str] = None
+    ) -> list:
         """
         Unpack a list from file.
 
@@ -355,7 +359,9 @@ class TuplePackager(ListPackager):
         """
         return super().pack_file(obj=list(obj), key=key, file_format=file_format)
 
-    def unpack_file(self, data_item: DataItem, file_format: str = None) -> tuple:
+    def unpack_file(
+        self, data_item: DataItem, file_format: Optional[str] = None
+    ) -> tuple:
         """
         Unpack a tuple from file.
 
@@ -400,7 +406,9 @@ class SetPackager(ListPackager):
         """
         return super().pack_file(obj=list(obj), key=key, file_format=file_format)
 
-    def unpack_file(self, data_item: DataItem, file_format: str = None) -> set:
+    def unpack_file(
+        self, data_item: DataItem, file_format: Optional[str] = None
+    ) -> set:
         """
         Unpack a set from file.
 
@@ -434,7 +442,9 @@ class FrozensetPackager(SetPackager):
         """
         return super().pack_file(obj=set(obj), key=key, file_format=file_format)
 
-    def unpack_file(self, data_item: DataItem, file_format: str = None) -> frozenset:
+    def unpack_file(
+        self, data_item: DataItem, file_format: Optional[str] = None
+    ) -> frozenset:
         """
         Unpack a frozenset from file.
 
@@ -481,7 +491,9 @@ class BytesPackager(ListPackager):
         """
         return super().pack_file(obj=list(obj), key=key, file_format=file_format)
 
-    def unpack_file(self, data_item: DataItem, file_format: str = None) -> bytes:
+    def unpack_file(
+        self, data_item: DataItem, file_format: Optional[str] = None
+    ) -> bytes:
         """
         Unpack a bytes from file.
 
@@ -526,7 +538,9 @@ class BytearrayPackager(BytesPackager):
         """
         return super().pack_file(obj=bytes(obj), key=key, file_format=file_format)
 
-    def unpack_file(self, data_item: DataItem, file_format: str = None) -> bytearray:
+    def unpack_file(
+        self, data_item: DataItem, file_format: Optional[str] = None
+    ) -> bytearray:
         """
         Unpack a bytearray from file.
 
@@ -585,7 +599,7 @@ class PathPackager(StrPackager):
         self,
         data_item: DataItem,
         is_directory: bool = False,
-        archive_format: str = None,
+        archive_format: Optional[str] = None,
     ) -> pathlib.Path:
         """
         Unpack a data item representing a `Path`. If the path is of a file, the file is downloaded to a local

mlrun/package/packagers_manager.py

@@ -17,10 +17,11 @@ import inspect
 import os
 import shutil
 import traceback
-from typing import Any, Union
+from typing import Any, Optional, Union
 
+import mlrun.errors
 from mlrun.artifacts import Artifact
-from mlrun.datastore import DataItem, store_manager
+from mlrun.datastore import DataItem, get_store_resource, store_manager
 from mlrun.errors import MLRunInvalidArgumentError
 from mlrun.utils import logger
 
@@ -41,7 +42,7 @@ class PackagersManager:
     It prepares the instructions / log hint configurations and then looks for the first packager that fits the task.
     """
 
-    def __init__(self, default_packager: type[Packager] = None):
+    def __init__(self, default_packager: Optional[type[Packager]] = None):
         """
         Initialize a packagers manager.
 
@@ -302,18 +303,17 @@ class PackagersManager:
 
     def link_packages(
         self,
-        additional_artifacts: list[Artifact],
+        additional_artifact_uris: dict,
         additional_results: dict,
     ):
         """
         Link packages to each other according to the provided extra data and metrics spec keys. A future link is
         marked with ellipses (...). If no link is found, None is used and a warning is printed.
 
-        :param additional_artifacts: Additional artifacts to link (should come from an `mlrun.MLClientCtx`).
-        :param additional_results:   Additional results to link (should come from an `mlrun.MLClientCtx`).
+        :param additional_artifact_uris:    Additional artifact URIs to link (should come from an `mlrun.MLClientCtx`).
+        :param additional_results:          Additional results to link (should come from an `mlrun.MLClientCtx`).
         """
         # Join the manager's artifacts and results with the additional ones to look for a link in all of them:
-        joined_artifacts = [*additional_artifacts, *self.artifacts]
         joined_results = {**additional_results, **self.results}
 
         # Go over the artifacts and link:
@@ -324,7 +324,10 @@ class PackagersManager:
                 if artifact.spec.extra_data[key] is ...:
                     # Look for an artifact or result with this key to link it:
                     extra_data = self._look_for_extra_data(
-                        key=key, artifacts=joined_artifacts, results=joined_results
+                        key=key,
+                        artifacts=self.artifacts,
+                        artifact_uris=additional_artifact_uris,
+                        results=joined_results,
                     )
                     # Print a warning if a link is missing:
                     if extra_data is None:
@@ -398,8 +401,8 @@ class PackagersManager:
     def _get_packager_for_packing(
         self,
         obj: Any,
-        artifact_type: str = None,
-        configurations: dict = None,
+        artifact_type: Optional[str] = None,
+        configurations: Optional[dict] = None,
     ) -> Union[Packager, None]:
         """
         Look for a packager that can pack the provided object as the provided artifact type.
@@ -426,7 +429,7 @@ class PackagersManager:
         self,
         data_item: Any,
         type_hint: type,
-        artifact_type: str = None,
+        artifact_type: Optional[str] = None,
     ) -> Union[Packager, None]:
         """
         Look for a packager that can unpack the data item of the given type hint as the provided artifact type.
@@ -664,16 +667,9 @@ class PackagersManager:
                         data_item=data_item,
                         instructions={},
                     )
-                except Exception as exception:
+                except Exception:
                     # Could not unpack as the reduced type hint, collect the exception and go to the next one:
-                    exception_string = "".join(
-                        traceback.format_exception(
-                            etype=type(exception),
-                            value=exception,
-                            tb=exception.__traceback__,
-                        )
-                    )
-                    found_packagers.append((packager, exception_string))
+                    found_packagers.append((packager, traceback.format_exc()))
             # Reduce the type hint list and continue:
             possible_type_hints = TypeHintUtils.reduce_type_hint(
                 type_hint=possible_type_hints
@@ -689,15 +685,8 @@ class PackagersManager:
                 artifact_type=None,
                 instructions={},
             )
-        except Exception as exception:
-            exception_string = "".join(
-                traceback.format_exception(
-                    etype=type(exception),
-                    value=exception,
-                    tb=exception.__traceback__,
-                )
-            )
-            found_packagers.append((self._default_packager, exception_string))
+        except Exception:
+            found_packagers.append((self._default_packager, traceback.format_exc()))
 
         # The method did not return until this point, raise an error:
         raise MLRunPackageUnpackingError(
@@ -715,17 +704,31 @@ class PackagersManager:
     def _look_for_extra_data(
         key: str,
         artifacts: list[Artifact],
+        artifact_uris: dict,
         results: dict,
     ) -> Union[Artifact, str, int, float, None]:
         """
         Look for an extra data item (artifact or result) by given key. If not found, None is returned.
 
-        :param key:       Key to look for.
-        :param artifacts: Artifacts to look in.
-        :param results:   Results to look in.
+        :param key:             Key to look for.
+        :param artifacts:       Artifacts to look in.
+        :param artifact_uris:   Artifacts URIs to look in.
+        :param results:         Results to look in.
 
         :return: The artifact or result with the same key or None if not found.
         """
+        artifact_uris = artifact_uris or {}
+        for _key, uri in artifact_uris.items():
+            if key == _key:
+                try:
+                    return get_store_resource(uri)
+                except mlrun.errors.MLRunNotFoundError as exc:
+                    logger.warn(
+                        f"Artifact {key=} not found when looking for extra data",
+                        exc=mlrun.errors.err_to_str(exc),
+                    )
+                    return None
+
         # Look in the artifacts:
         for artifact in artifacts:
             if key == artifact.key:

mlrun/package/utils/__init__.py

@@ -11,9 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
 from ._archiver import ArchiveSupportedFormat
 from ._formatter import StructFileSupportedFormat

mlrun/package/utils/_pickler.py

@@ -19,7 +19,7 @@ import sys
 import tempfile
 import warnings
 from types import ModuleType
-from typing import Any, Union
+from typing import Any, Optional, Union
 
 from mlrun.errors import MLRunInvalidArgumentError
 from mlrun.utils import logger
@@ -34,7 +34,7 @@ class Pickler:
 
     @staticmethod
     def pickle(
-        obj: Any, pickle_module_name: str, output_path: str = None
+        obj: Any, pickle_module_name: str, output_path: Optional[str] = None
     ) -> tuple[str, dict[str, Union[str, None]]]:
         """
         Pickle an object using the given module. The pickled object will be saved to file to the given output path.
@@ -91,10 +91,10 @@ class Pickler:
     def unpickle(
         pickle_path: str,
         pickle_module_name: str,
-        object_module_name: str = None,
-        python_version: str = None,
-        pickle_module_version: str = None,
-        object_module_version: str = None,
+        object_module_name: Optional[str] = None,
+        python_version: Optional[str] = None,
+        pickle_module_version: Optional[str] = None,
+        object_module_version: Optional[str] = None,
     ) -> Any:
         """
         Unpickle an object using the given instructions. Warnings may be raised in case any of the versions are

mlrun/platforms/__init__.py

@@ -12,23 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 import json
+import warnings
 from pprint import pprint
 from time import sleep
-
-from mlrun_pipelines.common.mounts import VolumeMount
-from mlrun_pipelines.mounts import (
-    auto_mount,
-    mount_configmap,
-    mount_hostpath,
-    mount_pvc,
-    mount_s3,
-    mount_secret,
-    mount_v3io,
-    set_env_variables,
-    v3io_cred,
-)
+from typing import Optional
 
 from .iguazio import (
     V3ioStreamClient,
@@ -37,10 +25,53 @@ from .iguazio import (
 )
 
 
+class _DeprecationHelper:
+    """A helper class to deprecate old schemas"""
+
+    def __init__(self, new_target: str, version="1.8.0"):
+        self._new_target = new_target
+        self._version = version
+
+    def __call__(self, *args, **kwargs):
+        self._warn()
+        return self._lazy_load()(*args, **kwargs)
+
+    def __getattr__(self, attr):
+        self._warn()
+        return getattr(self._lazy_load(), attr)
+
+    def _lazy_load(self, *args, **kwargs):
+        import mlrun.runtimes.mounts as mlrun_mounts
+
+        return getattr(mlrun_mounts, self._new_target)
+
+    def _warn(self):
+        warnings.warn(
+            f"mlrun.platforms.{self._new_target} is deprecated since version {self._version}, "
+            f"and will be removed in 1.10. Use mlrun.runtimes.mounts.{self._new_target} instead.",
+            FutureWarning,
+        )
+
+
+# TODO: Remove in 1.10
+# For backwards compatibility
+VolumeMount = _DeprecationHelper("VolumeMount")
+auto_mount = _DeprecationHelper("auto_mount")
+mount_configmap = _DeprecationHelper("mount_configmap")
+mount_hostpath = _DeprecationHelper("mount_hostpath")
+mount_pvc = _DeprecationHelper("mount_pvc")
+mount_s3 = _DeprecationHelper("mount_s3")
+mount_secret = _DeprecationHelper("mount_secret")
+mount_v3io = _DeprecationHelper("mount_v3io")
+set_env_variables = _DeprecationHelper("set_env_variables")
+v3io_cred = _DeprecationHelper("v3io_cred")
+# eof 'For backwards compatibility'
+
+
 def watch_stream(
     url,
-    shard_ids: list = None,
-    seek_to: str = None,
+    shard_ids: Optional[list] = None,
+    seek_to: Optional[str] = None,
     interval=None,
     is_json=False,
     **kwargs,

mlrun/platforms/iguazio.py

@@ -15,6 +15,7 @@
 import json
 import os
 import urllib
+from typing import Optional
 from urllib.parse import urlparse
 
 import requests
@@ -250,7 +251,9 @@ class KafkaOutputStream:
 
 
 class V3ioStreamClient:
-    def __init__(self, url: str, shard_id: int = 0, seek_to: str = None, **kwargs):
+    def __init__(
+        self, url: str, shard_id: int = 0, seek_to: Optional[str] = None, **kwargs
+    ):
         endpoint, stream_path = parse_path(url)
         seek_options = ["EARLIEST", "LATEST", "TIME", "SEQUENCE"]
         seek_to = seek_to or "LATEST"

mlrun/projects/operations.py

@@ -12,16 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import typing
 import warnings
 from typing import Optional, Union
 
-import mlrun_pipelines.common.models
-import mlrun_pipelines.models
-
 import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas.function
 import mlrun.common.schemas.workflow
+import mlrun_pipelines.common.models
+import mlrun_pipelines.models
 from mlrun.utils import hub_prefix
 
 from .pipelines import enrich_function_object, pipeline_context
@@ -59,25 +59,25 @@ def _get_engine_and_function(function, project=None):
 
 def run_function(
     function: Union[str, mlrun.runtimes.BaseRuntime],
-    handler: str = None,
+    handler: Optional[Union[str, typing.Callable]] = None,
     name: str = "",
-    params: dict = None,
-    hyperparams: dict = None,
+    params: Optional[dict] = None,
+    hyperparams: Optional[dict] = None,
     hyper_param_options: mlrun.model.HyperParamOptions = None,
-    inputs: dict = None,
-    outputs: list[str] = None,
+    inputs: Optional[dict] = None,
+    outputs: Optional[list[str]] = None,
     workdir: str = "",
-    labels: dict = None,
+    labels: Optional[dict] = None,
     base_task: mlrun.model.RunTemplate = None,
     watch: bool = True,
-    local: bool = None,
-    verbose: bool = None,
-    selector: str = None,
+    local: Optional[bool] = None,
+    verbose: Optional[bool] = None,
+    selector: Optional[str] = None,
     project_object=None,
-    auto_build: bool = None,
+    auto_build: Optional[bool] = None,
     schedule: Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
-    artifact_path: str = None,
-    notifications: list[mlrun.model.Notification] = None,
+    artifact_path: Optional[str] = None,
+    notifications: Optional[list[mlrun.model.Notification]] = None,
     returns: Optional[list[Union[str, dict[str, str]]]] = None,
     builder_env: Optional[list] = None,
     reset_on_run: Optional[bool] = None,
@@ -255,19 +255,19 @@ class BuildStatus:
 
 def build_function(
     function: Union[str, mlrun.runtimes.BaseRuntime],
-    with_mlrun: bool = None,
+    with_mlrun: Optional[bool] = None,
     skip_deployed: bool = False,
     image=None,
     base_image=None,
-    commands: list = None,
+    commands: Optional[list] = None,
     secret_name=None,
-    requirements: Union[str, list[str]] = None,
-    requirements_file: str = None,
+    requirements: Optional[Union[str, list[str]]] = None,
+    requirements_file: Optional[str] = None,
     mlrun_version_specifier=None,
-    builder_env: dict = None,
+    builder_env: Optional[dict] = None,
     project_object=None,
     overwrite_build_params: bool = False,
-    extra_args: str = None,
+    extra_args: Optional[str] = None,
     force_build: bool = False,
 ) -> Union[BuildStatus, mlrun_pipelines.models.PipelineNodeWrapper]:
     """deploy ML function, build container with its dependencies
@@ -294,9 +294,9 @@ def build_function(
     :param force_build: Force building the image, even when no changes were made
     """
     if not overwrite_build_params:
-        # TODO: change overwrite_build_params default to True in 1.8.0
+        # TODO: change overwrite_build_params default to True in 1.10.0
         warnings.warn(
-            "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.8.0.",
+            "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.10.0.",
             mlrun.utils.OverwriteBuildParamsWarning,
         )
 
@@ -325,7 +325,7 @@ def build_function(
             skip_deployed=skip_deployed,
         )
     else:
-        # TODO: remove filter once overwrite_build_params default is changed to True in 1.8.0
+        # TODO: remove filter once overwrite_build_params default is changed to True in 1.10.0
         with warnings.catch_warnings():
             warnings.simplefilter(
                 "ignore", category=mlrun.utils.OverwriteBuildParamsWarning
@@ -371,13 +371,13 @@ class DeployStatus:
 
 def deploy_function(
     function: Union[str, mlrun.runtimes.BaseRuntime],
-    models: list = None,
-    env: dict = None,
-    tag: str = None,
-    verbose: bool = None,
-    builder_env: dict = None,
+    models: Optional[list] = None,
+    env: Optional[dict] = None,
+    tag: Optional[str] = None,
+    verbose: Optional[bool] = None,
+    builder_env: Optional[dict] = None,
     project_object=None,
-    mock: bool = None,
+    mock: Optional[bool] = None,
 ) -> Union[DeployStatus, mlrun_pipelines.models.PipelineNodeWrapper]:
     """deploy real-time (nuclio based) functions