mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/package/utils/__init__.py

@@ -0,0 +1,53 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
+
+from ._archiver import ArchiveSupportedFormat
+from ._formatter import StructFileSupportedFormat
+from ._pickler import Pickler
+from ._supported_format import SupportedFormat
+from .log_hint_utils import LogHintKey, LogHintUtils
+from .type_hint_utils import TypeHintUtils
+
+# The default pickle module to use for pickling objects:
+DEFAULT_PICKLE_MODULE = "cloudpickle"
+# The default archive format to use for archiving directories:
+DEFAULT_ARCHIVE_FORMAT = ArchiveSupportedFormat.ZIP
+# The default struct file format to use for savings python struct objects (dictionaries and lists):
+DEFAULT_STRUCT_FILE_FORMAT = StructFileSupportedFormat.JSON
+
+
+class ArtifactType:
+    """
+    Possible artifact types to pack objects as and log using a `mlrun.Packager`.
+    """
+
+    OBJECT = "object"
+    PATH = "path"
+    FILE = "file"
+    DATASET = "dataset"
+    MODEL = "model"
+    PLOT = "plot"
+    RESULT = "result"
+
+
+class DatasetFileFormat:
+    """
+    All file format for logging objects as `DatasetArtifact`.
+    """
+
+    CSV = "csv"
+    PARQUET = "parquet"

mlrun/package/utils/_archiver.py

@@ -0,0 +1,226 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+import tarfile
+import zipfile
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from ._supported_format import SupportedFormat
+
+
+class _Archiver(ABC):
+    """
+    An abstract base class for an archiver - a class to manage archives of multiple files.
+    """
+
+    @classmethod
+    @abstractmethod
+    def create_archive(cls, directory_path: str, output_path: str) -> str:
+        """
+        Create an archive of all the contents in the given directory and save it to an archive file named as the
+        directory in the provided output path.
+
+        :param directory_path: The directory with the files to archive.
+        :param output_path:    The output path to store the created archive file.
+
+        :return: The created archive path.
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    def extract_archive(cls, archive_path: str, output_path: str) -> str:
+        """
+        Extract the given archive to a directory named as the archive file (without the extension) located in the
+        provided output path.
+
+        :param archive_path: The archive file to extract its contents.
+        :param output_path:  The output path to extract the directory of the archive to.
+
+        :return: The extracted contents directory path.
+        """
+        pass
+
+
+class _ZipArchiver(_Archiver):
+    """
+    A static class for managing zip archives.
+    """
+
+    @classmethod
+    def create_archive(cls, directory_path: str, output_path: str) -> str:
+        """
+        Create an archive of all the contents in the given directory and save it to an archive file named as the
+        directory in the provided output path.
+
+        :param directory_path: The directory with the files to archive.
+        :param output_path:    The output path to store the created archive file.
+
+        :return: The created archive path.
+        """
+        # Convert to `pathlib.Path` objects:
+        directory_path = Path(directory_path)
+        output_path = Path(output_path)
+
+        # Construct the archive file path:
+        archive_path = output_path / f"{directory_path.stem}.zip"
+
+        # Archive:
+        with zipfile.ZipFile(archive_path, "w") as zip_file:
+            for path in directory_path.rglob("*"):
+                zip_file.write(filename=path, arcname=path.relative_to(directory_path))
+
+        return str(archive_path)
+
+    @classmethod
+    def extract_archive(cls, archive_path: str, output_path: str) -> str:
+        """
+        Extract the given archive to a directory named as the archive file (without the extension) located in the
+        provided output path.
+
+        :param archive_path: The archive file to extract its contents.
+        :param output_path:  The output path to extract the directory of the archive to.
+
+        :return: The extracted contents directory path.
+        """
+        # Convert to `pathlib.Path` objects:
+        archive_path = Path(archive_path)
+        output_path = Path(output_path)
+
+        # Create the directory path:
+        directory_path = output_path / archive_path.stem
+        os.makedirs(directory_path)
+
+        # Extract:
+        with zipfile.ZipFile(archive_path, "r") as zip_file:
+            zip_file.extractall(directory_path)
+
+        return str(directory_path)
+
+
+class _TarArchiver(_Archiver):
+    """
+    A static class for managing tar archives.
+    """
+
+    # Inner class variable to note how to open a `TarFile` object for reading and writing:
+    _MODE_STRING = ""
+
+    @classmethod
+    def create_archive(cls, directory_path: str, output_path: str) -> str:
+        """
+        Create an archive of all the contents in the given directory and save it to an archive file named as the
+        directory in the provided output path.
+
+        :param directory_path: The directory with the files to archive.
+        :param output_path:    The output path to store the created archive file.
+
+        :return: The created archive path.
+        """
+        # Convert to `pathlib.Path` objects:
+        directory_path = Path(directory_path)
+        output_path = Path(output_path)
+
+        # Construct the archive file path:
+        archive_file_extension = (
+            "tar" if cls._MODE_STRING == "" else f"tar.{cls._MODE_STRING}"
+        )
+        archive_path = output_path / f"{directory_path.stem}.{archive_file_extension}"
+
+        # Archive:
+        with tarfile.open(archive_path, f"w:{cls._MODE_STRING}") as tar_file:
+            for path in directory_path.rglob("*"):
+                tar_file.add(name=path, arcname=path.relative_to(directory_path))
+
+        return str(archive_path)
+
+    @classmethod
+    def extract_archive(cls, archive_path: str, output_path: str) -> str:
+        """
+        Extract the given archive to a directory named as the archive file (without the extension) located in the
+        provided output path.
+
+        :param archive_path: The archive file to extract its contents.
+        :param output_path:  The output path to extract the directory of the archive to.
+
+        :return: The extracted contents directory path.
+        """
+        # Convert to `pathlib.Path` objects:
+        archive_path = Path(archive_path)
+        output_path = Path(output_path)
+
+        # Get the archive file name (can be constructed of multiple extensions like tar.gz so `Path.stem` won't work):
+        archive_file_name = archive_path
+        while archive_file_name.with_suffix(suffix="") != archive_file_name:
+            archive_file_name = archive_file_name.with_suffix(suffix="")
+        archive_file_name = archive_file_name.stem
+
+        # Create the directory path:
+        directory_path = output_path / archive_file_name
+        os.makedirs(directory_path)
+
+        # Extract:
+        with tarfile.open(archive_path, f"r:{cls._MODE_STRING}") as tar_file:
+            tar_file.extractall(directory_path)
+
+        return str(directory_path)
+
+
+class _TarGZArchiver(_TarArchiver):
+    """
+    A static class for managing tar.gz archives.
+    """
+
+    # Inner class variable to note how to open a `TarFile` object for reading and writing:
+    _MODE_STRING = "gz"
+
+
+class _TarBZ2Archiver(_TarArchiver):
+    """
+    A static class for managing tar.bz2 archives.
+    """
+
+    # Inner class variable to note how to open a `TarFile` object for reading and writing:
+    _MODE_STRING = "bz2"
+
+
+class _TarXZArchiver(_TarArchiver):
+    """
+    A static class for managing tar.gz archives.
+    """
+
+    # Inner class variable to note how to open a `TarFile` object for reading and writing:
+    _MODE_STRING = "xz"
+
+
+class ArchiveSupportedFormat(SupportedFormat[_Archiver]):
+    """
+    Library of archive formats (file extensions) supported by some builtin MLRun packagers.
+    """
+
+    ZIP = "zip"
+    TAR = "tar"
+    TAR_GZ = "tar.gz"
+    TAR_BZ2 = "tar.bz2"
+    TAR_XZ = "tar.xz"
+
+    _FORMAT_HANDLERS_MAP = {
+        ZIP: _ZipArchiver,
+        TAR: _TarArchiver,
+        TAR_GZ: _TarGZArchiver,
+        TAR_BZ2: _TarBZ2Archiver,
+        TAR_XZ: _TarXZArchiver,
+    }

mlrun/package/utils/_formatter.py

@@ -0,0 +1,211 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import ast
+import json
+from abc import ABC, abstractmethod
+from typing import Any, Union
+
+import yaml
+
+from ._supported_format import SupportedFormat
+
+
+class _Formatter(ABC):
+    """
+    An abstract base class for a formatter - a class to format python structures into and from files.
+    """
+
+    @classmethod
+    @abstractmethod
+    def write(cls, obj: Any, file_path: str, **dump_kwargs: dict):
+        """
+        Write the object to a file. The object must be serializable according to the used format.
+
+        :param obj:         The object to write.
+        :param file_path:   The file path to write to.
+        :param dump_kwargs: Additional keyword arguments to pass to the dump method of the formatter in use.
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    def read(cls, file_path: str) -> Any:
+        """
+        Read an object from the file given.
+
+        :param file_path: The file to read the object from.
+
+        :return: The read object.
+        """
+        pass
+
+
+class _JSONFormatter(_Formatter):
+    """
+    A static class for managing json files.
+    """
+
+    # A set of default configurations to pass to the dump function:
+    DEFAULT_DUMP_KWARGS = {"indent": 4}
+
+    @classmethod
+    def write(cls, obj: Union[list, dict], file_path: str, **dump_kwargs: dict):
+        """
+        Write the object to a json file. The object must be serializable according to the json format.
+
+        :param obj:         The object to write.
+        :param file_path:   The file path to write to.
+        :param dump_kwargs: Additional keyword arguments to pass to the `json.dump` method of the formatter in use.
+        """
+        dump_kwargs = dump_kwargs or cls.DEFAULT_DUMP_KWARGS
+        with open(file_path, "w") as file:
+            json.dump(obj, file, **dump_kwargs)
+
+    @classmethod
+    def read(cls, file_path: str) -> Union[list, dict]:
+        """
+        Read an object from the json file given.
+
+        :param file_path: The json file to read the object from.
+
+        :return: The read object.
+        """
+        with open(file_path, "r") as file:
+            obj = json.load(file)
+        return obj
+
+
+class _JSONLFormatter(_Formatter):
+    """
+    A static class for managing jsonl files.
+    """
+
+    @classmethod
+    def write(cls, obj: Union[list, dict], file_path: str, **dump_kwargs: dict):
+        """
+        Write the object to a jsonl file. The object must be serializable according to the json format.
+
+        :param obj:         The object to write.
+        :param file_path:   The file path to write to.
+        :param dump_kwargs: Additional keyword arguments to pass to the `json.dumps` method of the formatter in use.
+        """
+        if isinstance(obj, dict):
+            obj = [obj]
+
+        with open(file_path, "w") as file:
+            for line in obj:
+                file.write(json.dumps(obj=line, **dump_kwargs) + "\n")
+
+    @classmethod
+    def read(cls, file_path: str) -> Union[list, dict]:
+        """
+        Read an object from the jsonl file given.
+
+        :param file_path: The jsonl file to read the object from.
+
+        :return: The read object.
+        """
+        with open(file_path, "r") as file:
+            lines = file.readlines()
+
+        obj = []
+        for line in lines:
+            obj.append(json.loads(s=line))
+
+        return obj[0] if len(obj) == 1 else obj
+
+
+class _YAMLFormatter(_Formatter):
+    """
+    A static class for managing yaml files.
+    """
+
+    # A set of default configurations to pass to the dump function:
+    DEFAULT_DUMP_KWARGS = {"default_flow_style": False, "indent": 4}
+
+    @classmethod
+    def write(cls, obj: Union[list, dict], file_path: str, **dump_kwargs: dict):
+        """
+        Write the object to a yaml file. The object must be serializable according to the yaml format.
+
+        :param obj:         The object to write.
+        :param file_path:   The file path to write to.
+        :param dump_kwargs: Additional keyword arguments to pass to the `yaml.dump` method of the formatter in use.
+        """
+        dump_kwargs = dump_kwargs or cls.DEFAULT_DUMP_KWARGS
+        with open(file_path, "w") as file:
+            yaml.dump(obj, file, **dump_kwargs)
+
+    @classmethod
+    def read(cls, file_path: str) -> Union[list, dict]:
+        """
+        Read an object from the yaml file given.
+
+        :param file_path: The yaml file to read the object from.
+
+        :return: The read object.
+        """
+        with open(file_path, "r") as file:
+            obj = yaml.safe_load(file)
+        return obj
+
+
+class _TXTFormatter(_Formatter):
+    """
+    A static class for managing txt files.
+    """
+
+    @classmethod
+    def write(cls, obj: Any, file_path: str, **dump_kwargs: dict):
+        """
+        Write the object to a text file. The object must be serializable according to python's ast module.
+
+        :param obj:         The object to write.
+        :param file_path:   The file path to write to.
+        :param dump_kwargs: Ignored.
+        """
+        with open(file_path, "w") as file:
+            file.write(str(obj))
+
+    @classmethod
+    def read(cls, file_path: str) -> Any:
+        """
+        Read an object from the yaml file given.
+
+        :param file_path: The yaml file to read the object from.
+
+        :return: The read object.
+        """
+        with open(file_path, "r") as file:
+            obj = ast.literal_eval(file.read())
+        return obj
+
+
+class StructFileSupportedFormat(SupportedFormat[_Formatter]):
+    """
+    Library of struct formats (file extensions) supported by some builtin MLRun packagers.
+    """
+
+    JSON = "json"
+    JSONL = "jsonl"
+    YAML = "yaml"
+    TXT = "txt"
+
+    _FORMAT_HANDLERS_MAP = {
+        JSON: _JSONFormatter,
+        JSONL: _JSONLFormatter,
+        YAML: _YAMLFormatter,
+        TXT: _TXTFormatter,
+    }