mlrun 1.7.2rc3__py3-none-any.whl → 1.8.0rc1__py3-none-any.whl

mlrun/package/packagers/pandas_packagers.py

@@ -17,7 +17,7 @@ import os
 import pathlib
 import tempfile
 from abc import ABC, abstractmethod
-from typing import Any, Union
+from typing import Any, Optional, Union
 
 import pandas as pd
 
@@ -56,7 +56,7 @@ class _Formatter(ABC):
     @classmethod
     @abstractmethod
     def read(
-        cls, file_path: str, unflatten_kwargs: dict = None, **read_kwargs
+        cls, file_path: str, unflatten_kwargs: Optional[dict] = None, **read_kwargs
     ) -> pd.DataFrame:
         """
         Read the dataframe from the given file path.
@@ -173,7 +173,7 @@ class _ParquetFormatter(_Formatter):
 
     @classmethod
     def read(
-        cls, file_path: str, unflatten_kwargs: dict = None, **read_kwargs
+        cls, file_path: str, unflatten_kwargs: Optional[dict] = None, **read_kwargs
     ) -> pd.DataFrame:
         """
         Read the dataframe from the given parquet file path.
@@ -221,7 +221,7 @@ class _CSVFormatter(_Formatter):
 
     @classmethod
     def read(
-        cls, file_path: str, unflatten_kwargs: dict = None, **read_kwargs
+        cls, file_path: str, unflatten_kwargs: Optional[dict] = None, **read_kwargs
     ) -> pd.DataFrame:
         """
         Read the dataframe from the given csv file path.
@@ -275,7 +275,7 @@ class _H5Formatter(_Formatter):
 
     @classmethod
     def read(
-        cls, file_path: str, unflatten_kwargs: dict = None, **read_kwargs
+        cls, file_path: str, unflatten_kwargs: Optional[dict] = None, **read_kwargs
     ) -> pd.DataFrame:
         """
         Read the dataframe from the given h5 file path.
@@ -332,7 +332,7 @@ class _XMLFormatter(_Formatter):
 
     @classmethod
     def read(
-        cls, file_path: str, unflatten_kwargs: dict = None, **read_kwargs
+        cls, file_path: str, unflatten_kwargs: Optional[dict] = None, **read_kwargs
     ) -> pd.DataFrame:
         """
         Read the dataframe from the given xml file path.
@@ -391,7 +391,7 @@ class _XLSXFormatter(_Formatter):
 
     @classmethod
     def read(
-        cls, file_path: str, unflatten_kwargs: dict = None, **read_kwargs
+        cls, file_path: str, unflatten_kwargs: Optional[dict] = None, **read_kwargs
     ) -> pd.DataFrame:
         """
         Read the dataframe from the given xlsx file path.
@@ -449,7 +449,7 @@ class _HTMLFormatter(_Formatter):
 
     @classmethod
     def read(
-        cls, file_path: str, unflatten_kwargs: dict = None, **read_kwargs
+        cls, file_path: str, unflatten_kwargs: Optional[dict] = None, **read_kwargs
     ) -> pd.DataFrame:
         """
         Read dataframes from the given html file path.
@@ -510,7 +510,7 @@ class _JSONFormatter(_Formatter):
 
     @classmethod
     def read(
-        cls, file_path: str, unflatten_kwargs: dict = None, **read_kwargs
+        cls, file_path: str, unflatten_kwargs: Optional[dict] = None, **read_kwargs
     ) -> pd.DataFrame:
         """
         Read dataframes from the given json file path.
@@ -565,7 +565,7 @@ class _FeatherFormatter(_Formatter):
 
     @classmethod
     def read(
-        cls, file_path: str, unflatten_kwargs: dict = None, **read_kwargs
+        cls, file_path: str, unflatten_kwargs: Optional[dict] = None, **read_kwargs
     ) -> pd.DataFrame:
         """
         Read dataframes from the given feather file path.
@@ -620,7 +620,7 @@ class _ORCFormatter(_Formatter):
 
     @classmethod
     def read(
-        cls, file_path: str, unflatten_kwargs: dict = None, **read_kwargs
+        cls, file_path: str, unflatten_kwargs: Optional[dict] = None, **read_kwargs
     ) -> pd.DataFrame:
         """
         Read dataframes from the given orc file path.
@@ -730,7 +730,7 @@ class PandasDataFramePackager(DefaultPackager):
         self,
         obj: pd.DataFrame,
         key: str,
-        file_format: str = None,
+        file_format: Optional[str] = None,
         flatten: bool = True,
         **to_kwargs,
     ) -> tuple[Artifact, dict]:
@@ -785,8 +785,8 @@ class PandasDataFramePackager(DefaultPackager):
     def unpack_file(
         self,
         data_item: DataItem,
-        file_format: str = None,
-        read_kwargs: dict = None,
+        file_format: Optional[str] = None,
+        read_kwargs: Optional[dict] = None,
     ) -> pd.DataFrame:
         """
         Unpack a pandas dataframe from file.
@@ -883,7 +883,7 @@ class PandasSeriesPackager(PandasDataFramePackager):
         self,
         obj: pd.Series,
         key: str,
-        file_format: str = None,
+        file_format: Optional[str] = None,
         flatten: bool = True,
         **to_kwargs,
     ) -> tuple[Artifact, dict]:
@@ -919,9 +919,9 @@ class PandasSeriesPackager(PandasDataFramePackager):
     def unpack_file(
         self,
         data_item: DataItem,
-        file_format: str = None,
-        read_kwargs: dict = None,
-        column_name: Union[str, int] = None,
+        file_format: Optional[str] = None,
+        read_kwargs: Optional[dict] = None,
+        column_name: Optional[Union[str, int]] = None,
     ) -> pd.Series:
         """
         Unpack a pandas series from file.

mlrun/package/packagers/python_standard_library_packagers.py

@@ -15,7 +15,7 @@
 import os
 import pathlib
 import tempfile
-from typing import Union
+from typing import Optional, Union
 
 from mlrun.artifacts import Artifact
 from mlrun.datastore import DataItem
@@ -140,7 +140,7 @@ class StrPackager(DefaultPackager):
         self,
         data_item: DataItem,
         is_directory: bool = False,
-        archive_format: str = None,
+        archive_format: Optional[str] = None,
     ) -> str:
         """
         Unpack a data item representing a path string. If the path is of a file, the file is downloaded to a local
@@ -222,7 +222,7 @@ class _BuiltinCollectionPackager(DefaultPackager):
         return artifact, instructions
 
     def unpack_file(
-        self, data_item: DataItem, file_format: str = None
+        self, data_item: DataItem, file_format: Optional[str] = None
     ) -> Union[dict, list]:
         """
         Unpack a builtin collection from file.
@@ -259,7 +259,9 @@ class DictPackager(_BuiltinCollectionPackager):
 
     PACKABLE_OBJECT_TYPE = dict
 
-    def unpack_file(self, data_item: DataItem, file_format: str = None) -> dict:
+    def unpack_file(
+        self, data_item: DataItem, file_format: Optional[str] = None
+    ) -> dict:
         """
         Unpack a dictionary from file.
 
@@ -285,7 +287,9 @@ class ListPackager(_BuiltinCollectionPackager):
 
     PACKABLE_OBJECT_TYPE = list
 
-    def unpack_file(self, data_item: DataItem, file_format: str = None) -> list:
+    def unpack_file(
+        self, data_item: DataItem, file_format: Optional[str] = None
+    ) -> list:
         """
         Unpack a list from file.
 
@@ -355,7 +359,9 @@ class TuplePackager(ListPackager):
         """
         return super().pack_file(obj=list(obj), key=key, file_format=file_format)
 
-    def unpack_file(self, data_item: DataItem, file_format: str = None) -> tuple:
+    def unpack_file(
+        self, data_item: DataItem, file_format: Optional[str] = None
+    ) -> tuple:
         """
         Unpack a tuple from file.
 
@@ -400,7 +406,9 @@ class SetPackager(ListPackager):
         """
         return super().pack_file(obj=list(obj), key=key, file_format=file_format)
 
-    def unpack_file(self, data_item: DataItem, file_format: str = None) -> set:
+    def unpack_file(
+        self, data_item: DataItem, file_format: Optional[str] = None
+    ) -> set:
         """
         Unpack a set from file.
 
@@ -434,7 +442,9 @@ class FrozensetPackager(SetPackager):
         """
         return super().pack_file(obj=set(obj), key=key, file_format=file_format)
 
-    def unpack_file(self, data_item: DataItem, file_format: str = None) -> frozenset:
+    def unpack_file(
+        self, data_item: DataItem, file_format: Optional[str] = None
+    ) -> frozenset:
         """
         Unpack a frozenset from file.
 
@@ -481,7 +491,9 @@ class BytesPackager(ListPackager):
         """
         return super().pack_file(obj=list(obj), key=key, file_format=file_format)
 
-    def unpack_file(self, data_item: DataItem, file_format: str = None) -> bytes:
+    def unpack_file(
+        self, data_item: DataItem, file_format: Optional[str] = None
+    ) -> bytes:
         """
         Unpack a bytes from file.
 
@@ -526,7 +538,9 @@ class BytearrayPackager(BytesPackager):
         """
         return super().pack_file(obj=bytes(obj), key=key, file_format=file_format)
 
-    def unpack_file(self, data_item: DataItem, file_format: str = None) -> bytearray:
+    def unpack_file(
+        self, data_item: DataItem, file_format: Optional[str] = None
+    ) -> bytearray:
         """
         Unpack a bytearray from file.
 
@@ -585,7 +599,7 @@ class PathPackager(StrPackager):
         self,
         data_item: DataItem,
         is_directory: bool = False,
-        archive_format: str = None,
+        archive_format: Optional[str] = None,
     ) -> pathlib.Path:
         """
         Unpack a data item representing a `Path`. If the path is of a file, the file is downloaded to a local

mlrun/package/packagers_manager.py

@@ -17,10 +17,11 @@ import inspect
 import os
 import shutil
 import traceback
-from typing import Any, Union
+from typing import Any, Optional, Union
 
+import mlrun.errors
 from mlrun.artifacts import Artifact
-from mlrun.datastore import DataItem, store_manager
+from mlrun.datastore import DataItem, get_store_resource, store_manager
 from mlrun.errors import MLRunInvalidArgumentError
 from mlrun.utils import logger
 
@@ -41,7 +42,7 @@ class PackagersManager:
     It prepares the instructions / log hint configurations and then looks for the first packager that fits the task.
     """
 
-    def __init__(self, default_packager: type[Packager] = None):
+    def __init__(self, default_packager: Optional[type[Packager]] = None):
         """
         Initialize a packagers manager.
 
@@ -302,18 +303,17 @@ class PackagersManager:
 
     def link_packages(
         self,
-        additional_artifacts: list[Artifact],
+        additional_artifact_uris: dict,
         additional_results: dict,
     ):
         """
         Link packages to each other according to the provided extra data and metrics spec keys. A future link is
         marked with ellipses (...). If no link is found, None is used and a warning is printed.
 
-        :param additional_artifacts: Additional artifacts to link (should come from an `mlrun.MLClientCtx`).
-        :param additional_results:   Additional results to link (should come from an `mlrun.MLClientCtx`).
+        :param additional_artifact_uris:    Additional artifact URIs to link (should come from an `mlrun.MLClientCtx`).
+        :param additional_results:          Additional results to link (should come from an `mlrun.MLClientCtx`).
         """
         # Join the manager's artifacts and results with the additional ones to look for a link in all of them:
-        joined_artifacts = [*additional_artifacts, *self.artifacts]
         joined_results = {**additional_results, **self.results}
 
         # Go over the artifacts and link:
@@ -324,7 +324,10 @@ class PackagersManager:
                 if artifact.spec.extra_data[key] is ...:
                     # Look for an artifact or result with this key to link it:
                     extra_data = self._look_for_extra_data(
-                        key=key, artifacts=joined_artifacts, results=joined_results
+                        key=key,
+                        artifacts=self.artifacts,
+                        artifact_uris=additional_artifact_uris,
+                        results=joined_results,
                     )
                     # Print a warning if a link is missing:
                     if extra_data is None:
@@ -398,8 +401,8 @@ class PackagersManager:
     def _get_packager_for_packing(
         self,
         obj: Any,
-        artifact_type: str = None,
-        configurations: dict = None,
+        artifact_type: Optional[str] = None,
+        configurations: Optional[dict] = None,
     ) -> Union[Packager, None]:
         """
         Look for a packager that can pack the provided object as the provided artifact type.
@@ -426,7 +429,7 @@ class PackagersManager:
         self,
         data_item: Any,
         type_hint: type,
-        artifact_type: str = None,
+        artifact_type: Optional[str] = None,
     ) -> Union[Packager, None]:
         """
         Look for a packager that can unpack the data item of the given type hint as the provided artifact type.
@@ -715,17 +718,31 @@ class PackagersManager:
     def _look_for_extra_data(
         key: str,
         artifacts: list[Artifact],
+        artifact_uris: dict,
         results: dict,
     ) -> Union[Artifact, str, int, float, None]:
         """
         Look for an extra data item (artifact or result) by given key. If not found, None is returned.
 
-        :param key:       Key to look for.
-        :param artifacts: Artifacts to look in.
-        :param results:   Results to look in.
+        :param key:             Key to look for.
+        :param artifacts:       Artifacts to look in.
+        :param artifact_uris:   Artifacts URIs to look in.
+        :param results:         Results to look in.
 
         :return: The artifact or result with the same key or None if not found.
         """
+        artifact_uris = artifact_uris or {}
+        for _key, uri in artifact_uris.items():
+            if key == _key:
+                try:
+                    return get_store_resource(uri)
+                except mlrun.errors.MLRunNotFoundError as exc:
+                    logger.warn(
+                        f"Artifact {key=} not found when looking for extra data",
+                        exc=mlrun.errors.err_to_str(exc),
+                    )
+                    return None
+
         # Look in the artifacts:
         for artifact in artifacts:
             if key == artifact.key:

mlrun/package/utils/__init__.py

@@ -11,9 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
 from ._archiver import ArchiveSupportedFormat
 from ._formatter import StructFileSupportedFormat

mlrun/package/utils/_pickler.py

@@ -19,7 +19,7 @@ import sys
 import tempfile
 import warnings
 from types import ModuleType
-from typing import Any, Union
+from typing import Any, Optional, Union
 
 from mlrun.errors import MLRunInvalidArgumentError
 from mlrun.utils import logger
@@ -34,7 +34,7 @@ class Pickler:
 
     @staticmethod
     def pickle(
-        obj: Any, pickle_module_name: str, output_path: str = None
+        obj: Any, pickle_module_name: str, output_path: Optional[str] = None
     ) -> tuple[str, dict[str, Union[str, None]]]:
         """
         Pickle an object using the given module. The pickled object will be saved to file to the given output path.
@@ -91,10 +91,10 @@ class Pickler:
     def unpickle(
         pickle_path: str,
         pickle_module_name: str,
-        object_module_name: str = None,
-        python_version: str = None,
-        pickle_module_version: str = None,
-        object_module_version: str = None,
+        object_module_name: Optional[str] = None,
+        python_version: Optional[str] = None,
+        pickle_module_version: Optional[str] = None,
+        object_module_version: Optional[str] = None,
     ) -> Any:
         """
         Unpickle an object using the given instructions. Warnings may be raised in case any of the versions are

mlrun/platforms/__init__.py

@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 import json
 from pprint import pprint
 from time import sleep
+from typing import Optional
 
 from mlrun_pipelines.common.mounts import VolumeMount
 from mlrun_pipelines.mounts import (
@@ -39,8 +39,8 @@ from .iguazio import (
 
 def watch_stream(
     url,
-    shard_ids: list = None,
-    seek_to: str = None,
+    shard_ids: Optional[list] = None,
+    seek_to: Optional[str] = None,
     interval=None,
     is_json=False,
     **kwargs,

mlrun/platforms/iguazio.py

@@ -15,6 +15,7 @@
 import json
 import os
 import urllib
+from typing import Optional
 from urllib.parse import urlparse
 
 import requests
@@ -250,7 +251,9 @@ class KafkaOutputStream:
 
 
 class V3ioStreamClient:
-    def __init__(self, url: str, shard_id: int = 0, seek_to: str = None, **kwargs):
+    def __init__(
+        self, url: str, shard_id: int = 0, seek_to: Optional[str] = None, **kwargs
+    ):
         endpoint, stream_path = parse_path(url)
         seek_options = ["EARLIEST", "LATEST", "TIME", "SEQUENCE"]
         seek_to = seek_to or "LATEST"

mlrun/projects/__init__.py

@@ -27,12 +27,7 @@ __all__ = [
 ]
 
 from .operations import build_function, deploy_function, run_function  # noqa
-from .pipelines import (
-    import_remote_project,
-    load_and_run_workflow,
-    load_and_run,
-    pipeline_context,
-)  # noqa
+from .pipelines import load_and_run, pipeline_context  # noqa
 from .project import (
     MlrunProject,
     ProjectMetadata,

mlrun/projects/operations.py

@@ -12,16 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import typing
 import warnings
 from typing import Optional, Union
 
-import mlrun_pipelines.common.models
-import mlrun_pipelines.models
-
 import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas.function
 import mlrun.common.schemas.workflow
+import mlrun_pipelines.common.models
+import mlrun_pipelines.models
 from mlrun.utils import hub_prefix
 
 from .pipelines import enrich_function_object, pipeline_context
@@ -59,25 +59,25 @@ def _get_engine_and_function(function, project=None):
 
 def run_function(
     function: Union[str, mlrun.runtimes.BaseRuntime],
-    handler: str = None,
+    handler: Optional[Union[str, typing.Callable]] = None,
     name: str = "",
-    params: dict = None,
-    hyperparams: dict = None,
+    params: Optional[dict] = None,
+    hyperparams: Optional[dict] = None,
     hyper_param_options: mlrun.model.HyperParamOptions = None,
-    inputs: dict = None,
-    outputs: list[str] = None,
+    inputs: Optional[dict] = None,
+    outputs: Optional[list[str]] = None,
     workdir: str = "",
-    labels: dict = None,
+    labels: Optional[dict] = None,
     base_task: mlrun.model.RunTemplate = None,
     watch: bool = True,
-    local: bool = None,
-    verbose: bool = None,
-    selector: str = None,
+    local: Optional[bool] = None,
+    verbose: Optional[bool] = None,
+    selector: Optional[str] = None,
     project_object=None,
-    auto_build: bool = None,
+    auto_build: Optional[bool] = None,
     schedule: Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
-    artifact_path: str = None,
-    notifications: list[mlrun.model.Notification] = None,
+    artifact_path: Optional[str] = None,
+    notifications: Optional[list[mlrun.model.Notification]] = None,
     returns: Optional[list[Union[str, dict[str, str]]]] = None,
     builder_env: Optional[list] = None,
     reset_on_run: Optional[bool] = None,
@@ -255,19 +255,19 @@ class BuildStatus:
 
 def build_function(
     function: Union[str, mlrun.runtimes.BaseRuntime],
-    with_mlrun: bool = None,
+    with_mlrun: Optional[bool] = None,
     skip_deployed: bool = False,
     image=None,
     base_image=None,
-    commands: list = None,
+    commands: Optional[list] = None,
     secret_name=None,
-    requirements: Union[str, list[str]] = None,
-    requirements_file: str = None,
+    requirements: Optional[Union[str, list[str]]] = None,
+    requirements_file: Optional[str] = None,
     mlrun_version_specifier=None,
-    builder_env: dict = None,
+    builder_env: Optional[dict] = None,
     project_object=None,
     overwrite_build_params: bool = False,
-    extra_args: str = None,
+    extra_args: Optional[str] = None,
     force_build: bool = False,
 ) -> Union[BuildStatus, mlrun_pipelines.models.PipelineNodeWrapper]:
     """deploy ML function, build container with its dependencies
@@ -371,13 +371,13 @@ class DeployStatus:
 
 def deploy_function(
     function: Union[str, mlrun.runtimes.BaseRuntime],
-    models: list = None,
-    env: dict = None,
-    tag: str = None,
-    verbose: bool = None,
-    builder_env: dict = None,
+    models: Optional[list] = None,
+    env: Optional[dict] = None,
+    tag: Optional[str] = None,
+    verbose: Optional[bool] = None,
+    builder_env: Optional[dict] = None,
     project_object=None,
-    mock: bool = None,
+    mock: Optional[bool] = None,
 ) -> Union[DeployStatus, mlrun_pipelines.models.PipelineNodeWrapper]:
     """deploy real-time (nuclio based) functions