pydatamax 0.1.14__py3-none-any.whl → 0.1.15__py3-none-any.whl

datamax/loader/oss_handler.py

@@ -1,191 +1,191 @@
-import datetime
-import os
-import subprocess
-
-import oss2
-from dotenv import load_dotenv
-from loguru import logger
-from tqdm import tqdm
-
-load_dotenv()
-
-
-def removing(path):
-    for root, dirs, files in os.walk(path):
-        for dir in dirs:
-            if dir == "__pycache__":
-                pycache_path = os.path.join(root, dir)
-                subprocess.run(["rm", "-rf", pycache_path], check=False)
-
-
-def format_size_adaptive(value):
-    units = ["B", "KB", "MB", "GB", "TB", "PB"]
-    size = 1024.0
-    for i in range(len(units)):
-        if (value / size) < 1:
-            return "%.2f%s" % (value, units[i])
-        value = value / size
-
-
-def format_datetime_into_isoformat(date_time: datetime.datetime) -> str:
-    return (
-        date_time.replace(tzinfo=datetime.timezone.utc)
-        .isoformat()
-        .replace("+00:00", "Z")
-    )
-
-
-class OssClient:
-    def __init__(
-        self, oss_access_key_id, oss_access_key_secret, oss_endpoint, oss_bucket_name
-    ):
-        self.bucket_name = oss_bucket_name
-        self.auth = oss2.Auth(
-            os.getenv("OSS_ACCESS_KEY_ID", oss_access_key_id),
-            os.getenv("OSS_ACCESS_KEY_SECRET", oss_access_key_secret),
-        )
-        self.endpoint = os.getenv("OSS_ENDPOINT", oss_endpoint)
-        self.bucket = oss2.Bucket(
-            self.auth, self.endpoint, os.getenv("OSS_BUCKET_NAME", oss_bucket_name)
-        )
-
-    # Upload a file
-    # Usage: ossBucket.put_object_from_file("my-object-key", "path/to/local/file.txt")
-    def put_object_from_file(self, object_name, file_path, progress_callback=None):
-        self.bucket.put_object_from_file(
-            object_name, file_path, progress_callback=progress_callback
-        )
-
-    # Download a file
-    # Usage: ossBucket.get_object_to_file("my-object-key", "path/to/local/output-file.txt")
-    def get_object_to_file(self, object_name, file_path, progress_callback=None):
-        try:
-            self.bucket.get_object_to_file(
-                object_name, file_path, progress_callback=progress_callback
-            )
-        except oss2.exceptions.NoSuchKey:
-            raise
-
-            # Upload a folder
-
-    # Usage: ossBucket.put_object_from_folder("my-object-folder", "path/to/local/folder")
-    def put_pdf_word_from_folder(
-        self, object_folder_name, local_folder_path, progress_callback=None
-    ):
-        for root, dirs, files in os.walk(local_folder_path):
-            for file in tqdm(files, desc=root):
-                if file.endswith(".pdf") or file.endswith(".word"):
-                    file_path = os.path.join(root, file)
-                    object_name = os.path.join(
-                        object_folder_name, file_path[len(local_folder_path) + 1 :]
-                    )
-                    self.bucket.put_object_from_file(
-                        object_name, file_path, progress_callback=progress_callback
-                    )
-                    # logger.info("object name: {}, file path: {}".format(
-                    #     object_name, file_path))
-
-    # Upload a folder
-    # Usage: ossBucket.put_object_from_folder("my-object-folder", "path/to/local/folder")
-    def put_object_from_folder(
-        self, object_folder_name, local_folder_path, progress_callback=None
-    ):
-        for root, dirs, files in os.walk(local_folder_path):
-            for file in tqdm(files, desc=root):
-                file_path = os.path.join(root, file)
-                object_name = os.path.join(
-                    object_folder_name, file_path[len(local_folder_path) + 1 :]
-                )
-                self.bucket.put_object_from_file(
-                    object_name, file_path, progress_callback=progress_callback
-                )
-                logger.info(
-                    "object name: {}, file path: {}".format(object_name, file_path)
-                )
-
-    # Download a folder
-    # Usage: ossBucket.get_object_to_folder("my-object-folder", "path/to/local/output-folder")
-    def get_object_to_folder(
-        self, object_folder_name, local_folder_path, progress_callback=None
-    ):
-        os.makedirs(local_folder_path, exist_ok=True)
-        for obj in oss2.ObjectIterator(self.bucket, prefix=object_folder_name):
-            file_path = os.path.join(
-                local_folder_path, obj.key[len(object_folder_name) + 1 :]
-            )
-            self.bucket.get_object_to_file(
-                obj.key, file_path, progress_callback=progress_callback
-            )
-
-    # Get all objects in the bucket
-    # Usage: ossBucket.get_all_objects_in_bucket()
-    def get_all_objects_in_bucket(self, prefix=None, delimiter=None):
-        for obj in oss2.ObjectIterator(self.bucket, prefix=prefix, delimiter=delimiter):
-            if obj.is_prefix():  # obj is folder
-                logger.info("directory key: {}".format(obj.key))
-            else:  # obj is file
-                logger.info(
-                    "file key: {}, object last modified: {}, object size: {}".format(
-                        obj.key,
-                        format_datetime_into_isoformat(
-                            datetime.datetime.fromtimestamp(obj.last_modified)
-                        ),
-                        format_size_adaptive(obj.size),
-                    )
-                )
-
-    def get_objects_in_folders(self, prefix: str):
-        all_keys = []
-        for obj in oss2.ObjectIterator(self.bucket, prefix=prefix):
-            if obj.is_prefix():  # obj is folder
-                pass
-            else:  # obj is file
-                if obj.key.endswith("/"):
-                    continue
-                all_keys.append(obj.key)
-        return all_keys
-
-    def delete_object(self, object_name="test"):
-        if object_name is None or object_name == "":
-            raise Exception(
-                "Danger! object name is None or '' Will delete all objects in bucket!"
-            )
-        self.bucket.delete_object(key=object_name)
-
-    # Delete a folder
-    # Usage: ossBucket.delete_object_folder("my-object-folder")
-    def delete_object_folder(self, object_folder_name="test"):
-        if object_folder_name is None or object_folder_name == "":
-            raise Exception(
-                "Danger! object name is None or '' Will delete all objects in bucket!"
-            )
-        for obj in oss2.ObjectIterator(self.bucket, prefix=object_folder_name):
-            self.bucket.delete_object(obj.key)
-            logger.info("delete object key: {}".format(obj.key))
-
-    def get_oss_url(
-        self, object_name, url_expires_time, aliyun_oss_url_prefix, csnt_url_prefix
-    ):
-        oss_prefix = "oss://" + os.getenv("OSS_BUCKET_NAME", self.bucket_name) + "/"
-        if object_name.__contains__(oss_prefix):
-            object_name = object_name.replace(oss_prefix, "")
-        aliyun_url = self.bucket.sign_url(
-            "GET",
-            object_name,
-            int(os.getenv("URL_EXPIRES_TIME", url_expires_time)),
-            slash_safe=True,
-        )
-        csnt_url = aliyun_url.replace(
-            os.getenv("ALIYUN_OSS_URL_PREFIX", aliyun_oss_url_prefix),
-            os.getenv("CSNT_URL_PREFIX", csnt_url_prefix),
-        )
-        return csnt_url
-
-    def get_default_oss_url(self, object_name: str, url_expires_time):
-        aliyun_url = self.bucket.sign_url(
-            "GET",
-            object_name,
-            int(os.getenv("url_expires_time", url_expires_time)),
-            slash_safe=True,
-        )
-        return aliyun_url
+import datetime
+import os
+import subprocess
+
+import oss2
+from dotenv import load_dotenv
+from loguru import logger
+from tqdm import tqdm
+
+load_dotenv()
+
+
+def removing(path):
+    for root, dirs, files in os.walk(path):
+        for dir in dirs:
+            if dir == "__pycache__":
+                pycache_path = os.path.join(root, dir)
+                subprocess.run(["rm", "-rf", pycache_path], check=False)
+
+
+def format_size_adaptive(value):
+    units = ["B", "KB", "MB", "GB", "TB", "PB"]
+    size = 1024.0
+    for i in range(len(units)):
+        if (value / size) < 1:
+            return "%.2f%s" % (value, units[i])
+        value = value / size
+
+
+def format_datetime_into_isoformat(date_time: datetime.datetime) -> str:
+    return (
+        date_time.replace(tzinfo=datetime.timezone.utc)
+        .isoformat()
+        .replace("+00:00", "Z")
+    )
+
+
+class OssClient:
+    def __init__(
+        self, oss_access_key_id, oss_access_key_secret, oss_endpoint, oss_bucket_name
+    ):
+        self.bucket_name = oss_bucket_name
+        self.auth = oss2.Auth(
+            os.getenv("OSS_ACCESS_KEY_ID", oss_access_key_id),
+            os.getenv("OSS_ACCESS_KEY_SECRET", oss_access_key_secret),
+        )
+        self.endpoint = os.getenv("OSS_ENDPOINT", oss_endpoint)
+        self.bucket = oss2.Bucket(
+            self.auth, self.endpoint, os.getenv("OSS_BUCKET_NAME", oss_bucket_name)
+        )
+
+    # Upload a file
+    # Usage: ossBucket.put_object_from_file("my-object-key", "path/to/local/file.txt")
+    def put_object_from_file(self, object_name, file_path, progress_callback=None):
+        self.bucket.put_object_from_file(
+            object_name, file_path, progress_callback=progress_callback
+        )
+
+    # Download a file
+    # Usage: ossBucket.get_object_to_file("my-object-key", "path/to/local/output-file.txt")
+    def get_object_to_file(self, object_name, file_path, progress_callback=None):
+        try:
+            self.bucket.get_object_to_file(
+                object_name, file_path, progress_callback=progress_callback
+            )
+        except oss2.exceptions.NoSuchKey:
+            raise
+
+            # Upload a folder
+
+    # Usage: ossBucket.put_object_from_folder("my-object-folder", "path/to/local/folder")
+    def put_pdf_word_from_folder(
+        self, object_folder_name, local_folder_path, progress_callback=None
+    ):
+        for root, dirs, files in os.walk(local_folder_path):
+            for file in tqdm(files, desc=root):
+                if file.endswith(".pdf") or file.endswith(".word"):
+                    file_path = os.path.join(root, file)
+                    object_name = os.path.join(
+                        object_folder_name, file_path[len(local_folder_path) + 1 :]
+                    )
+                    self.bucket.put_object_from_file(
+                        object_name, file_path, progress_callback=progress_callback
+                    )
+                    # logger.info("object name: {}, file path: {}".format(
+                    #     object_name, file_path))
+
+    # Upload a folder
+    # Usage: ossBucket.put_object_from_folder("my-object-folder", "path/to/local/folder")
+    def put_object_from_folder(
+        self, object_folder_name, local_folder_path, progress_callback=None
+    ):
+        for root, dirs, files in os.walk(local_folder_path):
+            for file in tqdm(files, desc=root):
+                file_path = os.path.join(root, file)
+                object_name = os.path.join(
+                    object_folder_name, file_path[len(local_folder_path) + 1 :]
+                )
+                self.bucket.put_object_from_file(
+                    object_name, file_path, progress_callback=progress_callback
+                )
+                logger.info(
+                    "object name: {}, file path: {}".format(object_name, file_path)
+                )
+
+    # Download a folder
+    # Usage: ossBucket.get_object_to_folder("my-object-folder", "path/to/local/output-folder")
+    def get_object_to_folder(
+        self, object_folder_name, local_folder_path, progress_callback=None
+    ):
+        os.makedirs(local_folder_path, exist_ok=True)
+        for obj in oss2.ObjectIterator(self.bucket, prefix=object_folder_name):
+            file_path = os.path.join(
+                local_folder_path, obj.key[len(object_folder_name) + 1 :]
+            )
+            self.bucket.get_object_to_file(
+                obj.key, file_path, progress_callback=progress_callback
+            )
+
+    # Get all objects in the bucket
+    # Usage: ossBucket.get_all_objects_in_bucket()
+    def get_all_objects_in_bucket(self, prefix=None, delimiter=None):
+        for obj in oss2.ObjectIterator(self.bucket, prefix=prefix, delimiter=delimiter):
+            if obj.is_prefix():  # obj is folder
+                logger.info("directory key: {}".format(obj.key))
+            else:  # obj is file
+                logger.info(
+                    "file key: {}, object last modified: {}, object size: {}".format(
+                        obj.key,
+                        format_datetime_into_isoformat(
+                            datetime.datetime.fromtimestamp(obj.last_modified)
+                        ),
+                        format_size_adaptive(obj.size),
+                    )
+                )
+
+    def get_objects_in_folders(self, prefix: str):
+        all_keys = []
+        for obj in oss2.ObjectIterator(self.bucket, prefix=prefix):
+            if obj.is_prefix():  # obj is folder
+                pass
+            else:  # obj is file
+                if obj.key.endswith("/"):
+                    continue
+                all_keys.append(obj.key)
+        return all_keys
+
+    def delete_object(self, object_name="test"):
+        if object_name is None or object_name == "":
+            raise Exception(
+                "Danger! object name is None or '' Will delete all objects in bucket!"
+            )
+        self.bucket.delete_object(key=object_name)
+
+    # Delete a folder
+    # Usage: ossBucket.delete_object_folder("my-object-folder")
+    def delete_object_folder(self, object_folder_name="test"):
+        if object_folder_name is None or object_folder_name == "":
+            raise Exception(
+                "Danger! object name is None or '' Will delete all objects in bucket!"
+            )
+        for obj in oss2.ObjectIterator(self.bucket, prefix=object_folder_name):
+            self.bucket.delete_object(obj.key)
+            logger.info("delete object key: {}".format(obj.key))
+
+    def get_oss_url(
+        self, object_name, url_expires_time, aliyun_oss_url_prefix, csnt_url_prefix
+    ):
+        oss_prefix = "oss://" + os.getenv("OSS_BUCKET_NAME", self.bucket_name) + "/"
+        if object_name.__contains__(oss_prefix):
+            object_name = object_name.replace(oss_prefix, "")
+        aliyun_url = self.bucket.sign_url(
+            "GET",
+            object_name,
+            int(os.getenv("URL_EXPIRES_TIME", url_expires_time)),
+            slash_safe=True,
+        )
+        csnt_url = aliyun_url.replace(
+            os.getenv("ALIYUN_OSS_URL_PREFIX", aliyun_oss_url_prefix),
+            os.getenv("CSNT_URL_PREFIX", csnt_url_prefix),
+        )
+        return csnt_url
+
+    def get_default_oss_url(self, object_name: str, url_expires_time):
+        aliyun_url = self.bucket.sign_url(
+            "GET",
+            object_name,
+            int(os.getenv("url_expires_time", url_expires_time)),
+            slash_safe=True,
+        )
+        return aliyun_url

datamax/parser/__init__.py

@@ -1,4 +1,2 @@
-from .core import DataMax
-import logging
-logger = logging.getLogger()
-logger.addHandler(logging.NullHandler())
+from .core import DataMax
+from loguru import logger

datamax/parser/base.py

@@ -1,77 +1,77 @@
-import os
-from datetime import datetime
-from pathlib import Path
-from typing import List, Dict
-from datamax.utils.tokenizer import DashScopeClient
-
-
-class LifeCycle:
-    """
-    Life cycle class
-    """
-
-    def __init__(self, update_time: str, life_type: list, life_metadata: Dict[str, str]):
-        self.update_time = update_time  # Update time
-        self.life_type = life_type  # Life cycle type
-        self.life_metadata = life_metadata  # Life cycle metadata
-
-    def update(self, update_time: str, life_type: list, life_metadata: Dict[str, str]):
-        self.update_time = update_time
-        self.life_type = life_type
-        self.life_metadata.update(life_metadata)
-
-    def __str__(self):
-        metadata_str = ', '.join(f'{k}: {v}' for k, v in self.life_metadata.items())
-        return f'update_time: {self.update_time}, life_type: {self.life_type}, life_metadata: {{{metadata_str}}}'
-
-    def to_dict(self):
-        return {
-            'update_time': self.update_time,
-            'life_type': self.life_type,
-            'life_metadata': self.life_metadata
-        }
-
-
-class MarkdownOutputVo:
-    """
-    Markdown output conversion
-    """
-
-    def __init__(self, title: str, content: str):
-        self.title: str = title  # File type
-        self.content: str = content  # Markdown content
-        self.lifecycle: List[LifeCycle] = []  # Life cycle data
-
-    def add_lifecycle(self, lifecycle: LifeCycle):
-        self.lifecycle.append(lifecycle)
-
-    def to_dict(self):
-        data_dict = {
-            'title': self.title,
-            'content': self.content,
-            'lifecycle': [lc.to_dict() for lc in self.lifecycle]
-        }
-        return data_dict
-
-
-class BaseLife:
-    tk_client = DashScopeClient()
-
-    @staticmethod
-    def generate_lifecycle(source_file, domain, life_type, usage_purpose) -> LifeCycle:
-        update_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        life_type = [life_type]
-        storage = os.stat(source_file)
-        life_metadata = {
-            # "token_count": token_count,  # Token count of the text
-            "storage_size": storage.st_size,  # Storage size in bytes
-            "source_file": source_file,  # Source file
-            "domain": domain,  # Domain
-            "usage_purpose": usage_purpose  # Usage purpose
-        }
-        return LifeCycle(update_time, life_type, life_metadata)
-
-    @staticmethod
-    def get_file_extension(file_path):
-        file_path = Path(file_path)
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import List, Dict
+from datamax.utils.tokenizer import DashScopeClient
+
+
+class LifeCycle:
+    """
+    Life cycle class
+    """
+
+    def __init__(self, update_time: str, life_type: list, life_metadata: Dict[str, str]):
+        self.update_time = update_time  # Update time
+        self.life_type = life_type  # Life cycle type
+        self.life_metadata = life_metadata  # Life cycle metadata
+
+    def update(self, update_time: str, life_type: list, life_metadata: Dict[str, str]):
+        self.update_time = update_time
+        self.life_type = life_type
+        self.life_metadata.update(life_metadata)
+
+    def __str__(self):
+        metadata_str = ', '.join(f'{k}: {v}' for k, v in self.life_metadata.items())
+        return f'update_time: {self.update_time}, life_type: {self.life_type}, life_metadata: {{{metadata_str}}}'
+
+    def to_dict(self):
+        return {
+            'update_time': self.update_time,
+            'life_type': self.life_type,
+            'life_metadata': self.life_metadata
+        }
+
+
+class MarkdownOutputVo:
+    """
+    Markdown output conversion
+    """
+
+    def __init__(self, title: str, content: str):
+        self.title: str = title  # File type
+        self.content: str = content  # Markdown content
+        self.lifecycle: List[LifeCycle] = []  # Life cycle data
+
+    def add_lifecycle(self, lifecycle: LifeCycle):
+        self.lifecycle.append(lifecycle)
+
+    def to_dict(self):
+        data_dict = {
+            'title': self.title,
+            'content': self.content,
+            'lifecycle': [lc.to_dict() for lc in self.lifecycle]
+        }
+        return data_dict
+
+
+class BaseLife:
+    tk_client = DashScopeClient()
+
+    @staticmethod
+    def generate_lifecycle(source_file, domain, life_type, usage_purpose) -> LifeCycle:
+        update_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        life_type = [life_type]
+        storage = os.stat(source_file)
+        life_metadata = {
+            # "token_count": token_count,  # Token count of the text
+            "storage_size": storage.st_size,  # Storage size in bytes
+            "source_file": source_file,  # Source file
+            "domain": domain,  # Domain
+            "usage_purpose": usage_purpose  # Usage purpose
+        }
+        return LifeCycle(update_time, life_type, life_metadata)
+
+    @staticmethod
+    def get_file_extension(file_path):
+        file_path = Path(file_path)
         return file_path.suffix[1:].lower()