pydatamax 0.1.13__py3-none-any.whl → 0.1.15__py3-none-any.whl

datamax/__init__.py

@@ -1 +1 @@
-from .parser import DataMax
+from .parser import DataMax

datamax/loader/core.py

@@ -1,119 +1,119 @@
-import os
-from typing import List
-from datamax.loader.MinioHandler import MinIOClient
-from datamax.loader.OssHandler import OssClient
-
-
-class DataLoader:
-    def __init__(self, endpoint: str = None, secret_key: str = None, access_key: str = None,
-                 bucket_name: str = None, source: str = None):
-        if source and source == 'Oss':
-            self.oss = OssClient(
-                oss_endpoint=endpoint,
-                oss_access_key_secret=secret_key,
-                oss_access_key_id=access_key,
-                oss_bucket_name=bucket_name
-            )
-        elif source and source == 'MinIO':
-            self.mi = MinIOClient(
-                endpoint=endpoint,
-                secret_key=secret_key,
-                access_key=access_key,
-                bucket_name=bucket_name
-            )
-        self.download_path = str('./download_file')
-        self.source = source
-        self.bucket_name = bucket_name
-
-    @staticmethod
-    def load_from_file(local_file_path) -> List[str]:
-        if os.path.isfile(local_file_path):
-            if os.path.exists(local_file_path):
-                if os.access(local_file_path, os.R_OK):
-                    return [local_file_path]
-                else:
-                    return []
-            else:
-                return []
-        elif os.path.isdir(local_file_path):
-            access_path = []
-            for root, dirs, files in os.walk(local_file_path):
-                for file in files:
-                    file_path = os.path.join(root, file)
-                    if os.path.exists(file_path):
-                        if os.access(file_path, os.R_OK):
-                            access_path.append(file_path)
-                        else:
-                            continue
-                    else:
-                        continue
-            return access_path
-        else:
-            return []
-
-    def load_from_oss_source(self, oss_path: str) -> List[str]:
-        if not os.path.exists(self.download_path):
-            os.makedirs(self.download_path)
-
-        self.download(oss_path=oss_path)
-
-        file_list = []
-        for root, dirs, files in os.walk(self.download_path):
-            for file in files:
-                file_path = os.path.join(self.download_path, file)
-                file_list.append(file_path)
-
-        success_file_list = []
-        for file_path in file_list:
-            if self.load_from_file(file_path):
-                success_file_list.append(file_path)
-
-        return success_file_list
-
-    def download(self, oss_path: str):
-        if self.source == 'MinIO':
-            file_list = self.mi.list_objects(bucket_name=self.bucket_name, prefix=oss_path)
-            for path in file_list:
-                self.mi.download_file(bucket_name=self.bucket_name, object_name=path,
-                                      file_path=f'{self.download_path}/{path.split("/")[-1]}')
-        elif self.source == "Oss":
-            keys = self.oss.get_objects_in_folders(prefix=oss_path)
-            for path in keys:
-                self.oss.get_object_to_file(object_name=path,
-                                            file_path=f'{self.download_path}/{path.split("/")[-1]}')
-
-    def upload(self, local_file_path: str, save_prefix: str):
-        if self.source == 'MinIO':
-            if os.path.isdir(local_file_path):
-                for root, dirs, files in os.walk(local_file_path):
-                    for file in files:
-                        file_path = os.path.join(root, file)
-                        self.mi.upload_file(bucket_name=self.bucket_name, object_name=save_prefix + f'{file}',
-                                            file_path=file_path)
-            elif os.path.isfile(local_file_path):
-                self.mi.upload_file(bucket_name=self.bucket_name,
-                                    object_name=save_prefix + os.path.basename(local_file_path),
-                                    file_path=local_file_path)
-            else:
-                pass
-
-        elif self.source == "Oss":
-            if os.path.isdir(local_file_path):
-                self.oss.put_object_from_folder(object_folder_name=save_prefix, local_folder_path=local_file_path)
-            elif os.path.isfile(local_file_path):
-                self.oss.put_object_from_file(object_name=save_prefix + os.path.basename(local_file_path),
-                                              file_path=local_file_path)
-        else:
-            pass
-
-    def share(self, oss_path: str,
-              expires: int = None,
-              aliyun_oss_url_prefix: str = None,
-              csnt_url_prefix: str = None):
-        if self.source == 'MinIO':
-            return self.mi.get_object_tmp_link(bucket_name=self.bucket_name, object_name=oss_path, expires=expires)
-        elif self.source == "Oss":
-            return self.oss.get_oss_url(object_name=oss_path,
-                                        url_expires_time=expires,
-                                        aliyun_oss_url_prefix=aliyun_oss_url_prefix,
+import os
+from typing import List
+from datamax.loader.minio_handler import MinIOClient
+from datamax.loader.oss_handler import OssClient
+
+
+class DataLoader:
+    def __init__(self, endpoint: str = None, secret_key: str = None, access_key: str = None,
+                 bucket_name: str = None, source: str = None):
+        if source and source == 'Oss':
+            self.oss = OssClient(
+                oss_endpoint=endpoint,
+                oss_access_key_secret=secret_key,
+                oss_access_key_id=access_key,
+                oss_bucket_name=bucket_name
+            )
+        elif source and source == 'MinIO':
+            self.mi = MinIOClient(
+                endpoint=endpoint,
+                secret_key=secret_key,
+                access_key=access_key,
+                bucket_name=bucket_name
+            )
+        self.download_path = str('./download_file')
+        self.source = source
+        self.bucket_name = bucket_name
+
+    @staticmethod
+    def load_from_file(local_file_path) -> List[str]:
+        if os.path.isfile(local_file_path):
+            if os.path.exists(local_file_path):
+                if os.access(local_file_path, os.R_OK):
+                    return [local_file_path]
+                else:
+                    return []
+            else:
+                return []
+        elif os.path.isdir(local_file_path):
+            access_path = []
+            for root, dirs, files in os.walk(local_file_path):
+                for file in files:
+                    file_path = os.path.join(root, file)
+                    if os.path.exists(file_path):
+                        if os.access(file_path, os.R_OK):
+                            access_path.append(file_path)
+                        else:
+                            continue
+                    else:
+                        continue
+            return access_path
+        else:
+            return []
+
+    def load_from_oss_source(self, oss_path: str) -> List[str]:
+        if not os.path.exists(self.download_path):
+            os.makedirs(self.download_path)
+
+        self.download(oss_path=oss_path)
+
+        file_list = []
+        for root, dirs, files in os.walk(self.download_path):
+            for file in files:
+                file_path = os.path.join(self.download_path, file)
+                file_list.append(file_path)
+
+        success_file_list = []
+        for file_path in file_list:
+            if self.load_from_file(file_path):
+                success_file_list.append(file_path)
+
+        return success_file_list
+
+    def download(self, oss_path: str):
+        if self.source == 'MinIO':
+            file_list = self.mi.list_objects(bucket_name=self.bucket_name, prefix=oss_path)
+            for path in file_list:
+                self.mi.download_file(bucket_name=self.bucket_name, object_name=path,
+                                      file_path=f'{self.download_path}/{path.split("/")[-1]}')
+        elif self.source == "Oss":
+            keys = self.oss.get_objects_in_folders(prefix=oss_path)
+            for path in keys:
+                self.oss.get_object_to_file(object_name=path,
+                                            file_path=f'{self.download_path}/{path.split("/")[-1]}')
+
+    def upload(self, local_file_path: str, save_prefix: str):
+        if self.source == 'MinIO':
+            if os.path.isdir(local_file_path):
+                for root, dirs, files in os.walk(local_file_path):
+                    for file in files:
+                        file_path = os.path.join(root, file)
+                        self.mi.upload_file(bucket_name=self.bucket_name, object_name=save_prefix + f'{file}',
+                                            file_path=file_path)
+            elif os.path.isfile(local_file_path):
+                self.mi.upload_file(bucket_name=self.bucket_name,
+                                    object_name=save_prefix + os.path.basename(local_file_path),
+                                    file_path=local_file_path)
+            else:
+                pass
+
+        elif self.source == "Oss":
+            if os.path.isdir(local_file_path):
+                self.oss.put_object_from_folder(object_folder_name=save_prefix, local_folder_path=local_file_path)
+            elif os.path.isfile(local_file_path):
+                self.oss.put_object_from_file(object_name=save_prefix + os.path.basename(local_file_path),
+                                              file_path=local_file_path)
+        else:
+            pass
+
+    def share(self, oss_path: str,
+              expires: int = None,
+              aliyun_oss_url_prefix: str = None,
+              csnt_url_prefix: str = None):
+        if self.source == 'MinIO':
+            return self.mi.get_object_tmp_link(bucket_name=self.bucket_name, object_name=oss_path, expires=expires)
+        elif self.source == "Oss":
+            return self.oss.get_oss_url(object_name=oss_path,
+                                        url_expires_time=expires,
+                                        aliyun_oss_url_prefix=aliyun_oss_url_prefix,
                                         csnt_url_prefix=csnt_url_prefix)

datamax/loader/{MinioHandler.py → minio_handler.py}

@@ -1,172 +1,172 @@
-import os
-from dotenv import load_dotenv
-from datetime import timedelta
-from minio import Minio
-from minio.commonconfig import Tags
-from minio.error import S3Error
-from loguru import logger
-import re
-
-load_dotenv()
-
-
-class MinIOClient:
-    def __init__(self, endpoint, access_key, secret_key, bucket_name, secure=False):
-        self.endpoint = os.getenv("ENDPOINT_MINIO", endpoint)
-        self.access_key = os.getenv("ACCESS_KEY_MINIO", access_key)
-        self.secret_key = os.getenv("SECRET_KEY_MINIO", secret_key)
-        self.bucket_name = os.getenv("BUCKET_NAME_MINIO", bucket_name)
-        self.secure = secure
-        self.client = self._initialize_client()
-
-    def _initialize_client(self):
-        try:
-            client = Minio(
-                self.endpoint,
-                access_key=self.access_key,
-                secret_key=self.secret_key,
-                secure=self.secure
-            )
-            return client
-        except S3Error as e:
-            raise
-
-    @staticmethod
-    def bytes_to_mb(bytes_value):
-        return bytes_value / (1024 * 1024)
-
-    def create_bucket(self, bucket_name):
-        if self.client:
-            try:
-                self.client.make_bucket(bucket_name)
-                logger.info(f"Bucket '{bucket_name}' created successfully.")
-            except S3Error as e:
-                raise
-
-    def remove_bucket(self, bucket_name):
-        if self.client:
-            try:
-                self.client.remove_bucket(bucket_name)
-                logger.info(f"Bucket '{bucket_name}' removed successfully.")
-            except S3Error as e:
-                raise
-
-    def upload_file(self, file_path, bucket_name, object_name):
-        if self.client:
-            try:
-                self.client.fput_object(bucket_name, object_name, file_path)
-                logger.info(f"File '{file_path}' uploaded to bucket '{bucket_name}' as '{object_name}'.")
-            except S3Error as e:
-                raise
-
-    def download_file(self, bucket_name, object_name, file_path):
-        if self.client:
-            try:
-                self.client.fget_object(bucket_name, object_name, file_path)
-                logger.info(f"Object '{object_name}' from bucket '{bucket_name}' downloaded to '{file_path}'.")
-                return file_path
-            except Exception as e:
-                try:
-                    illegal_chars = r'[\/:*?"<>|]'
-                    file_path = re.sub(illegal_chars, '_', file_path)
-                    self.client.fget_object(bucket_name, object_name, file_path)
-                    logger.info(
-                        f"Object {object_name} from bucket {bucket_name} downloaded to {file_path}'.")
-                    return file_path
-                except Exception as e:
-                    raise
-
-    def list_objects(self, bucket_name, prefix=None):
-        if self.client:
-            try:
-                result_list = []
-                if prefix:
-                    objects = self.client.list_objects(bucket_name, recursive=True, prefix=prefix)
-                else:
-                    objects = self.client.list_objects(bucket_name, recursive=True)
-                logger.info(f"Objects in bucket '{bucket_name}':")
-                for obj in objects:
-                    result_list.append(obj.object_name)
-                return result_list
-            except S3Error as e:
-                raise
-
-    def remove_object(self, bucket_name, object_name):
-        if self.client:
-            try:
-                self.client.remove_object(bucket_name, object_name)
-            except S3Error as e:
-                raise
-
-    def calculate_bucket_stats(self, bucket_name, prefix):
-        objects = self.client.list_objects(bucket_name,
-                                           prefix=prefix, recursive=True)
-        total_size = 0
-        object_count = 0
-
-        for obj in objects:
-            object_count += 1
-            total_size += obj.size
-
-        total_size = self.bytes_to_mb(total_size)
-
-        return object_count, total_size
-
-    def get_objects(self, bucket_name, object_name):
-        try:
-            response = self.client.get_object(bucket_name, object_name)
-            content = response.read().decode('utf-8')
-            return content
-        except Exception as e:
-            raise
-
-    def get_object_tag(self, bucket_name, object_name):
-        try:
-            tags = self.client.get_object_tags(bucket_name=bucket_name, object_name=object_name)
-            return tags
-        except Exception as e:
-            raise
-
-    def update_object_tag(self, bucket_name, object_name, tags):
-        try:
-            tags_obj = Tags.new_object_tags()
-            tag_info = self.get_object_tag(bucket_name=bucket_name, object_name=object_name)
-            if tag_info is None:
-                tag_info = {}
-                for tag_dict in tags:
-                    for tag_key, tag_value in tag_dict.items():
-                        if tag_key in tag_info:
-                            tag_info[tag_key] = tag_value
-                        else:
-                            tag_info[tag_key] = tag_value
-
-                for k, v in tag_info.items():
-                    tags_obj[k] = v
-                self.client.set_object_tags(bucket_name=bucket_name, object_name=object_name, tags=tags_obj)
-            else:
-                for tag_dict in tags:
-                    for tag_key, tag_value in tag_dict.items():
-                        if tag_key in tag_info:
-                            tag_info[tag_key] = tag_value
-                        else:
-                            tag_info[tag_key] = tag_value
-
-                for k, v in tag_info.items():
-                    tags_obj[k] = v
-                self.client.set_object_tags(bucket_name=bucket_name, object_name=object_name, tags=tags_obj)
-            return tag_info
-        except Exception as e:
-            raise
-
-    def reset_object_tag(self, bucket_name, object_name):
-        try:
-            self.client.delete_object_tags(bucket_name=bucket_name, object_name=object_name)
-            return True
-        except Exception as e:
-            raise
-
-    def get_object_tmp_link(self, bucket_name, object_name, expires):
-        try:
-            return self.client.presigned_get_object(bucket_name, object_name, expires=timedelta(days=expires))
-        except Exception as e:
+import os
+from dotenv import load_dotenv
+from datetime import timedelta
+from minio import Minio
+from minio.commonconfig import Tags
+from minio.error import S3Error
+from loguru import logger
+import re
+
+load_dotenv()
+
+
+class MinIOClient:
+    def __init__(self, endpoint, access_key, secret_key, bucket_name, secure=False):
+        self.endpoint = os.getenv("ENDPOINT_MINIO", endpoint)
+        self.access_key = os.getenv("ACCESS_KEY_MINIO", access_key)
+        self.secret_key = os.getenv("SECRET_KEY_MINIO", secret_key)
+        self.bucket_name = os.getenv("BUCKET_NAME_MINIO", bucket_name)
+        self.secure = secure
+        self.client = self._initialize_client()
+
+    def _initialize_client(self):
+        try:
+            client = Minio(
+                self.endpoint,
+                access_key=self.access_key,
+                secret_key=self.secret_key,
+                secure=self.secure
+            )
+            return client
+        except S3Error as e:
+            raise
+
+    @staticmethod
+    def bytes_to_mb(bytes_value):
+        return bytes_value / (1024 * 1024)
+
+    def create_bucket(self, bucket_name):
+        if self.client:
+            try:
+                self.client.make_bucket(bucket_name)
+                logger.info(f"Bucket '{bucket_name}' created successfully.")
+            except S3Error as e:
+                raise
+
+    def remove_bucket(self, bucket_name):
+        if self.client:
+            try:
+                self.client.remove_bucket(bucket_name)
+                logger.info(f"Bucket '{bucket_name}' removed successfully.")
+            except S3Error as e:
+                raise
+
+    def upload_file(self, file_path, bucket_name, object_name):
+        if self.client:
+            try:
+                self.client.fput_object(bucket_name, object_name, file_path)
+                logger.info(f"File '{file_path}' uploaded to bucket '{bucket_name}' as '{object_name}'.")
+            except S3Error as e:
+                raise
+
+    def download_file(self, bucket_name, object_name, file_path):
+        if self.client:
+            try:
+                self.client.fget_object(bucket_name, object_name, file_path)
+                logger.info(f"Object '{object_name}' from bucket '{bucket_name}' downloaded to '{file_path}'.")
+                return file_path
+            except Exception as e:
+                try:
+                    illegal_chars = r'[\/:*?"<>|]'
+                    file_path = re.sub(illegal_chars, '_', file_path)
+                    self.client.fget_object(bucket_name, object_name, file_path)
+                    logger.info(
+                        f"Object {object_name} from bucket {bucket_name} downloaded to {file_path}'.")
+                    return file_path
+                except Exception as e:
+                    raise
+
+    def list_objects(self, bucket_name, prefix=None):
+        if self.client:
+            try:
+                result_list = []
+                if prefix:
+                    objects = self.client.list_objects(bucket_name, recursive=True, prefix=prefix)
+                else:
+                    objects = self.client.list_objects(bucket_name, recursive=True)
+                logger.info(f"Objects in bucket '{bucket_name}':")
+                for obj in objects:
+                    result_list.append(obj.object_name)
+                return result_list
+            except S3Error as e:
+                raise
+
+    def remove_object(self, bucket_name, object_name):
+        if self.client:
+            try:
+                self.client.remove_object(bucket_name, object_name)
+            except S3Error as e:
+                raise
+
+    def calculate_bucket_stats(self, bucket_name, prefix):
+        objects = self.client.list_objects(bucket_name,
+                                           prefix=prefix, recursive=True)
+        total_size = 0
+        object_count = 0
+
+        for obj in objects:
+            object_count += 1
+            total_size += obj.size
+
+        total_size = self.bytes_to_mb(total_size)
+
+        return object_count, total_size
+
+    def get_objects(self, bucket_name, object_name):
+        try:
+            response = self.client.get_object(bucket_name, object_name)
+            content = response.read().decode('utf-8')
+            return content
+        except Exception as e:
+            raise
+
+    def get_object_tag(self, bucket_name, object_name):
+        try:
+            tags = self.client.get_object_tags(bucket_name=bucket_name, object_name=object_name)
+            return tags
+        except Exception as e:
+            raise
+
+    def update_object_tag(self, bucket_name, object_name, tags):
+        try:
+            tags_obj = Tags.new_object_tags()
+            tag_info = self.get_object_tag(bucket_name=bucket_name, object_name=object_name)
+            if tag_info is None:
+                tag_info = {}
+                for tag_dict in tags:
+                    for tag_key, tag_value in tag_dict.items():
+                        if tag_key in tag_info:
+                            tag_info[tag_key] = tag_value
+                        else:
+                            tag_info[tag_key] = tag_value
+
+                for k, v in tag_info.items():
+                    tags_obj[k] = v
+                self.client.set_object_tags(bucket_name=bucket_name, object_name=object_name, tags=tags_obj)
+            else:
+                for tag_dict in tags:
+                    for tag_key, tag_value in tag_dict.items():
+                        if tag_key in tag_info:
+                            tag_info[tag_key] = tag_value
+                        else:
+                            tag_info[tag_key] = tag_value
+
+                for k, v in tag_info.items():
+                    tags_obj[k] = v
+                self.client.set_object_tags(bucket_name=bucket_name, object_name=object_name, tags=tags_obj)
+            return tag_info
+        except Exception as e:
+            raise
+
+    def reset_object_tag(self, bucket_name, object_name):
+        try:
+            self.client.delete_object_tags(bucket_name=bucket_name, object_name=object_name)
+            return True
+        except Exception as e:
+            raise
+
+    def get_object_tmp_link(self, bucket_name, object_name, expires):
+        try:
+            return self.client.presigned_get_object(bucket_name, object_name, expires=timedelta(days=expires))
+        except Exception as e:
             raise