dtlpy 1.114.17__py3-none-any.whl → 1.116.6__py3-none-any.whl

dtlpy/miscellaneous/list_print.py

@@ -1,105 +1,105 @@
-import datetime
-import tabulate
-import typing
-import logging
-import pandas
-
-from .. import exceptions
-
-logger = logging.getLogger(name='dtlpy')
-
-T = typing.TypeVar('T')
-
-
-class List(list, typing.MutableSequence[T]):
-    def to_df(self, show_all=False, columns=None):
-        try:
-            to_print = list()
-            keys_list = list()
-            for element in self.__iter__():
-                if hasattr(element, 'to_json'):
-                    item_dict = element.to_json()
-                else:
-                    item_dict = element
-                to_print.append(item_dict)
-                [keys_list.append(key) for key in list(item_dict.keys()) if key not in keys_list]
-            try:
-                # try sorting bt creation date
-                to_print = sorted(to_print, key=lambda k: k['createdAt'] if k['createdAt'] is not None else "")
-            except KeyError:
-                pass
-            except Exception:
-                logger.exception('Error sorting printing:')
-
-            remove_keys_list = ['contributors', 'url', 'annotations', 'items', 'export', 'directoryTree', 'org',
-                                '_contributors', 'role', 'account', 'featureConstraints',
-                                'attributes', 'partitions', 'metadata', 'stream', 'updatedAt', 'arch',
-                                'input', 'revisions', 'pipeline',  # task fields
-                                'feedbackQueue',  # session fields
-                                '_ontology_ids', '_labels',  # dataset
-                                'esInstance', 'esIndex',  # time series fields
-                                'thumbnail',  # item thumnail too long
-                                # services fields
-                                'driverId', 'useUserJwt', 'versions', 'runtime', 'mq', 'global',
-                                # triggers
-                                'scope',
-                                # Package
-                                'modules'
-                                ]
-            if not show_all:
-                if columns is not None:
-                    # take columns from inputs
-                    if not isinstance(columns, list):
-                        if not isinstance(columns, str):
-                            raise exceptions.PlatformException(
-                                error='3002',
-                                message='"columns" input must be str or list. found: {}'.format(type(columns)))
-                        columns = [columns]
-                    keys_list = columns
-                else:
-                    # take default columns
-                    for key in remove_keys_list:
-                        if key in keys_list:
-                            keys_list.remove(key)
-
-            for element in to_print:
-                # handle printing errors for not ascii string when in cli
-                if 'name' in element:
-                    try:
-                        # check if ascii
-                        element['name'].encode('ascii')
-                    except UnicodeEncodeError:
-                        # if not - print bytes instead
-                        element['name'] = str(element['name']).encode('utf-8')
-                if 'createdAt' in element:
-                    try:
-                        str_timestamp = str(element['createdAt'])
-                        if len(str_timestamp) > 10:
-                            str_timestamp = str_timestamp[:10]
-                        element['createdAt'] = datetime.datetime.fromtimestamp(int(str_timestamp), datetime.timezone.utc).isoformat()
-                    except Exception:
-                        pass
-            df = pandas.DataFrame(to_print, columns=keys_list)
-            return df
-        except Exception:
-            raise exceptions.PlatformException(error='3002',
-                                               message='Failed converting to DataFrame')
-
-    def print(self, show_all=False, level='print', to_return=False, columns=None):
-        try:
-            df = self.to_df(show_all=show_all, columns=columns)
-            if 'name' in list(df.columns.values):
-                df['name'] = df['name'].astype(str)
-
-            if to_return:
-                return tabulate.tabulate(df, headers='keys', tablefmt='psql')
-            else:
-                if level == 'print':
-                    print('\n{}'.format(tabulate.tabulate(df, headers='keys', tablefmt='psql')))
-                elif level == 'debug':
-                    logger.debug('\n{}'.format(tabulate.tabulate(df, headers='keys', tablefmt='psql')))
-                else:
-                    raise ValueError('unknown log level in printing: {}'.format(level))
-
-        except Exception:
-            raise exceptions.PlatformException(error='3002', message='Failed printing entity')
+import datetime
+import tabulate
+import typing
+import logging
+import pandas
+
+from .. import exceptions
+
+logger = logging.getLogger(name='dtlpy')
+
+T = typing.TypeVar('T')
+
+
+class List(list, typing.MutableSequence[T]):
+    def to_df(self, show_all=False, columns=None):
+        try:
+            to_print = list()
+            keys_list = list()
+            for element in self.__iter__():
+                if hasattr(element, 'to_json'):
+                    item_dict = element.to_json()
+                else:
+                    item_dict = element
+                to_print.append(item_dict)
+                [keys_list.append(key) for key in list(item_dict.keys()) if key not in keys_list]
+            try:
+                # try sorting bt creation date
+                to_print = sorted(to_print, key=lambda k: k['createdAt'] if k['createdAt'] is not None else "")
+            except KeyError:
+                pass
+            except Exception:
+                logger.exception('Error sorting printing:')
+
+            remove_keys_list = ['contributors', 'url', 'annotations', 'items', 'export', 'directoryTree', 'org',
+                                '_contributors', 'role', 'account', 'featureConstraints',
+                                'attributes', 'partitions', 'metadata', 'stream', 'updatedAt', 'arch',
+                                'input', 'revisions', 'pipeline',  # task fields
+                                'feedbackQueue',  # session fields
+                                '_ontology_ids', '_labels',  # dataset
+                                'esInstance', 'esIndex',  # time series fields
+                                'thumbnail',  # item thumnail too long
+                                # services fields
+                                'driverId', 'useUserJwt', 'versions', 'runtime', 'mq', 'global',
+                                # triggers
+                                'scope',
+                                # Package
+                                'modules'
+                                ]
+            if not show_all:
+                if columns is not None:
+                    # take columns from inputs
+                    if not isinstance(columns, list):
+                        if not isinstance(columns, str):
+                            raise exceptions.PlatformException(
+                                error='3002',
+                                message='"columns" input must be str or list. found: {}'.format(type(columns)))
+                        columns = [columns]
+                    keys_list = columns
+                else:
+                    # take default columns
+                    for key in remove_keys_list:
+                        if key in keys_list:
+                            keys_list.remove(key)
+
+            for element in to_print:
+                # handle printing errors for not ascii string when in cli
+                if 'name' in element:
+                    try:
+                        # check if ascii
+                        element['name'].encode('ascii')
+                    except UnicodeEncodeError:
+                        # if not - print bytes instead
+                        element['name'] = str(element['name']).encode('utf-8')
+                if 'createdAt' in element:
+                    try:
+                        str_timestamp = str(element['createdAt'])
+                        if len(str_timestamp) > 10:
+                            str_timestamp = str_timestamp[:10]
+                        element['createdAt'] = datetime.datetime.fromtimestamp(int(str_timestamp), datetime.timezone.utc).isoformat()
+                    except Exception:
+                        pass
+            df = pandas.DataFrame(to_print, columns=keys_list)
+            return df
+        except Exception:
+            raise exceptions.PlatformException(error='3002',
+                                               message='Failed converting to DataFrame')
+
+    def print(self, show_all=False, level='print', to_return=False, columns=None):
+        try:
+            df = self.to_df(show_all=show_all, columns=columns)
+            if 'name' in list(df.columns.values):
+                df['name'] = df['name'].astype(str)
+
+            if to_return:
+                return tabulate.tabulate(df, headers='keys', tablefmt='psql')
+            else:
+                if level == 'print':
+                    print('\n{}'.format(tabulate.tabulate(df, headers='keys', tablefmt='psql')))
+                elif level == 'debug':
+                    logger.debug('\n{}'.format(tabulate.tabulate(df, headers='keys', tablefmt='psql')))
+                else:
+                    raise ValueError('unknown log level in printing: {}'.format(level))
+
+        except Exception:
+            raise exceptions.PlatformException(error='3002', message='Failed printing entity')

dtlpy/miscellaneous/zipping.py

@@ -1,130 +1,130 @@
-import logging
-import os
-import zipfile
-from typing import List
-
-import numpy as np
-import pathspec
-
-logger = logging.getLogger(name='dtlpy')
-
-MAX_ZIP_FILE = 100e6  # 100MB
-
-
-class Zipping:
-    def __init__(self):
-        pass
-
-    @staticmethod
-    def zip_directory(zip_filename, directory=None, ignore_max_file_size=False, ignore_directories: List[str] = None):
-        """
-        Zip Directory
-        Will ignore .gitignore files
-
-        :param directory: the directory to zip
-        :param zip_filename: the name of the zipfile.
-        :param ignore_max_file_size: ignore the limitation on the zip file size
-        :param list[str] ignore_directories: directories to ignore.
-        :return: None
-        """
-        # default path
-        if directory is None:
-            directory = os.getcwd()
-        # check if directory
-        assert os.path.isdir(directory), '[ERROR] Directory does not exists: {}'.format(directory)
-
-        if '.gitignore' in os.listdir(directory):
-            with open(os.path.join(directory, '.gitignore')) as f:
-                spec_src = f.read()
-        else:
-            spec_src = ''
-        ignore_lines = spec_src.splitlines() + ['.git', '.dataloop']
-        if ignore_directories is not None:
-            ignore_lines += ignore_directories
-        spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, ignore_lines)
-
-        # init zip file
-        zip_file = zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED)
-        try:
-            for root, dirs, files in os.walk(directory):
-                # remove dirs to avoid going file by file
-                for d in dirs:
-                    if spec.match_file(os.path.relpath(os.path.join(root, d), directory)):
-                        dirs.remove(d)
-                for file in files:
-                    filepath = os.path.join(root, file)
-                    if not spec.match_file(os.path.relpath(filepath, directory)):
-                        Zipping.__add_to_zip_file(directory, filepath, ignore_max_file_size, zip_file)
-        finally:
-            zip_file.close()
-
-    @staticmethod
-    def zip_directory_inclusive(zip_filename, directory=None, ignore_max_file_size=False,
-                                subpaths: List[str] = None):
-        """
-        Zip Directory
-        Will ignore .gitignore files
-
-        :param directory: the directory to zip.
-        :param zip_filename: the name of the zipfile
-        :param ignore_max_file_size: ignore the limitation on the zip file size
-        :param list[str] subpaths: paths to include in the final zip (relative path).
-        :return: None
-        """
-        # default path
-        if directory is None:
-            directory = os.getcwd()
-        # check if directory
-        assert os.path.isdir(directory), '[ERROR] Directory does not exists: %s' % directory
-
-        if '.gitignore' in os.listdir(directory):
-            with open(os.path.join(directory, '.gitignore')) as f:
-                spec_src = f.read()
-        else:
-            spec_src = ''
-        ignore_lines = spec_src.splitlines() + ['.git', '.dataloop']
-        spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, ignore_lines)
-
-        # init zip file
-        zip_file = zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED)
-        try:
-            for root, dirs, files in os.walk(directory):
-                for file in files:
-                    filepath = os.path.join(root, file)
-                    if not spec.match_file(os.path.relpath(filepath, directory)) \
-                            and Zipping.__check_filepath(os.path.relpath(filepath, directory), subpaths):
-                        Zipping.__add_to_zip_file(directory, filepath, ignore_max_file_size, zip_file)
-        finally:
-            zip_file.close()
-
-    @staticmethod
-    def __check_filepath(filepath: str, paths: List[str]):
-        """
-        Checks whether a specific file is inside one of the subdirectories
-        """
-        return any(filepath.startswith(directory) for directory in paths)
-
-    @staticmethod
-    def __add_to_zip_file(directory, filepath, ignore_max_file_size, zip_file):
-        zip_file.write(filepath, arcname=os.path.relpath(filepath, directory))
-        if not ignore_max_file_size:
-            if np.sum([f.file_size for f in list(zip_file.NameToInfo.values())]) > MAX_ZIP_FILE:
-                logger.error('Failed zipping in file: {}'.format(filepath))
-                raise ValueError(
-                    'Zip file cant be over 100MB. '
-                    'Please verify that only code is being uploaded or '
-                    'add files to .gitignore so they wont be zipped and uploaded as code.')
-
-    @staticmethod
-    def unzip_directory(zip_filename, to_directory=None):
-        with zipfile.ZipFile(zip_filename) as zipdata:
-            zipinfos = zipdata.infolist()
-            # iterate through each file
-            for zipinfo in zipinfos:
-                # encode the file names
-                # zip package make decode by cp437 for file that have name that not ascii
-                # this happen when the flag_bits be different than 0
-                # so we encode the name back
-                if not zipinfo.flag_bits:
-                    zipinfo.filename = zipinfo.filename.encode('cp437').decode('utf-8')
-                zipdata.extract(zipinfo, to_directory)
+import logging
+import os
+import zipfile
+from typing import List
+
+import numpy as np
+import pathspec
+
+logger = logging.getLogger(name='dtlpy')
+
+MAX_ZIP_FILE = 100e6  # 100MB
+
+
+class Zipping:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def zip_directory(zip_filename, directory=None, ignore_max_file_size=False, ignore_directories: List[str] = None):
+        """
+        Zip Directory
+        Will ignore .gitignore files
+
+        :param directory: the directory to zip
+        :param zip_filename: the name of the zipfile.
+        :param ignore_max_file_size: ignore the limitation on the zip file size
+        :param list[str] ignore_directories: directories to ignore.
+        :return: None
+        """
+        # default path
+        if directory is None:
+            directory = os.getcwd()
+        # check if directory
+        assert os.path.isdir(directory), '[ERROR] Directory does not exists: {}'.format(directory)
+
+        if '.gitignore' in os.listdir(directory):
+            with open(os.path.join(directory, '.gitignore')) as f:
+                spec_src = f.read()
+        else:
+            spec_src = ''
+        ignore_lines = spec_src.splitlines() + ['.git', '.dataloop']
+        if ignore_directories is not None:
+            ignore_lines += ignore_directories
+        spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, ignore_lines)
+
+        # init zip file
+        zip_file = zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED)
+        try:
+            for root, dirs, files in os.walk(directory):
+                # remove dirs to avoid going file by file
+                for d in dirs:
+                    if spec.match_file(os.path.relpath(os.path.join(root, d), directory)):
+                        dirs.remove(d)
+                for file in files:
+                    filepath = os.path.join(root, file)
+                    if not spec.match_file(os.path.relpath(filepath, directory)):
+                        Zipping.__add_to_zip_file(directory, filepath, ignore_max_file_size, zip_file)
+        finally:
+            zip_file.close()
+
+    @staticmethod
+    def zip_directory_inclusive(zip_filename, directory=None, ignore_max_file_size=False,
+                                subpaths: List[str] = None):
+        """
+        Zip Directory
+        Will ignore .gitignore files
+
+        :param directory: the directory to zip.
+        :param zip_filename: the name of the zipfile
+        :param ignore_max_file_size: ignore the limitation on the zip file size
+        :param list[str] subpaths: paths to include in the final zip (relative path).
+        :return: None
+        """
+        # default path
+        if directory is None:
+            directory = os.getcwd()
+        # check if directory
+        assert os.path.isdir(directory), '[ERROR] Directory does not exists: %s' % directory
+
+        if '.gitignore' in os.listdir(directory):
+            with open(os.path.join(directory, '.gitignore')) as f:
+                spec_src = f.read()
+        else:
+            spec_src = ''
+        ignore_lines = spec_src.splitlines() + ['.git', '.dataloop']
+        spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, ignore_lines)
+
+        # init zip file
+        zip_file = zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED)
+        try:
+            for root, dirs, files in os.walk(directory):
+                for file in files:
+                    filepath = os.path.join(root, file)
+                    if not spec.match_file(os.path.relpath(filepath, directory)) \
+                            and Zipping.__check_filepath(os.path.relpath(filepath, directory), subpaths):
+                        Zipping.__add_to_zip_file(directory, filepath, ignore_max_file_size, zip_file)
+        finally:
+            zip_file.close()
+
+    @staticmethod
+    def __check_filepath(filepath: str, paths: List[str]):
+        """
+        Checks whether a specific file is inside one of the subdirectories
+        """
+        return any(filepath.startswith(directory) for directory in paths)
+
+    @staticmethod
+    def __add_to_zip_file(directory, filepath, ignore_max_file_size, zip_file):
+        zip_file.write(filepath, arcname=os.path.relpath(filepath, directory))
+        if not ignore_max_file_size:
+            if np.sum([f.file_size for f in list(zip_file.NameToInfo.values())]) > MAX_ZIP_FILE:
+                logger.error('Failed zipping in file: {}'.format(filepath))
+                raise ValueError(
+                    'Zip file cant be over 100MB. '
+                    'Please verify that only code is being uploaded or '
+                    'add files to .gitignore so they wont be zipped and uploaded as code.')
+
+    @staticmethod
+    def unzip_directory(zip_filename, to_directory=None):
+        with zipfile.ZipFile(zip_filename) as zipdata:
+            zipinfos = zipdata.infolist()
+            # iterate through each file
+            for zipinfo in zipinfos:
+                # encode the file names
+                # zip package make decode by cp437 for file that have name that not ascii
+                # this happen when the flag_bits be different than 0
+                # so we encode the name back
+                if not zipinfo.flag_bits:
+                    zipinfo.filename = zipinfo.filename.encode('cp437').decode('utf-8')
+                zipdata.extract(zipinfo, to_directory)

dtlpy/ml/__init__.py

@@ -1,20 +1,20 @@
-#! /usr/bin/env python3
-# This file is part of DTLPY.
-#
-# DTLPY is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# DTLPY is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with DTLPY.  If not, see <http://www.gnu.org/licenses/>.
-from .base_model_adapter import BaseModelAdapter
-
-from . import metrics
-from . import predictions_utils
-from . import train_utils
+#! /usr/bin/env python3
+# This file is part of DTLPY.
+#
+# DTLPY is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# DTLPY is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with DTLPY.  If not, see <http://www.gnu.org/licenses/>.
+from .base_model_adapter import BaseModelAdapter
+
+from . import metrics
+from . import predictions_utils
+from . import train_utils

dtlpy/ml/base_feature_extractor_adapter.py

@@ -1,28 +1,28 @@
-from abc import ABC
-
-from base_model_adapter import BaseModelAdapter
-from .. import entities
-
-
-class BaseFeatureExtractorAdapter(BaseModelAdapter, ABC):
-    def __int__(self, model_entity: entities.Model = None):
-        super().__init__(model_entity)
-
-    def extract_features(self, batch: list, **kwargs):
-        """ Runs inference with the model, but does not predict. Instead, extracts features for the input batch.
-
-            Virtual method - need to implement
-
-        :param batch: `list` a list containing a batch of items whose features will be extracted
-        """
-        raise NotImplementedError("Please implement 'extract_features' method in {}".format(self.__class__.__name__))
-
-    def extract_dataset_features(self, dataset: entities.Dataset, **kwargs):
-        """ Runs inference to extract features for all items in a dataset.
-
-            Virtual method - need to implement
-
-        :param dataset: `entities.Dataset` dataset entity whose items will have their features extracted
-        """
-        raise NotImplementedError("Please implement 'extract_dataset_features' method in "
+from abc import ABC
+
+from base_model_adapter import BaseModelAdapter
+from .. import entities
+
+
+class BaseFeatureExtractorAdapter(BaseModelAdapter, ABC):
+    def __int__(self, model_entity: entities.Model = None):
+        super().__init__(model_entity)
+
+    def extract_features(self, batch: list, **kwargs):
+        """ Runs inference with the model, but does not predict. Instead, extracts features for the input batch.
+
+            Virtual method - need to implement
+
+        :param batch: `list` a list containing a batch of items whose features will be extracted
+        """
+        raise NotImplementedError("Please implement 'extract_features' method in {}".format(self.__class__.__name__))
+
+    def extract_dataset_features(self, dataset: entities.Dataset, **kwargs):
+        """ Runs inference to extract features for all items in a dataset.
+
+            Virtual method - need to implement
+
+        :param dataset: `entities.Dataset` dataset entity whose items will have their features extracted
+        """
+        raise NotImplementedError("Please implement 'extract_dataset_features' method in "
                                   "{}".format(self.__class__.__name__))