pydatamax 0.1.5__py3-none-any.whl → 0.1.12__py3-none-any.whl

datamax/__init__.py

@@ -1 +1 @@
-from .parser import DataMaxParser
+from .parser import DataMax

datamax/loader/OssHandler.py

@@ -1,9 +1,11 @@
+import datetime
 import os
+import subprocess
+
 import oss2
-import datetime
-from tqdm import tqdm
-from loguru import logger
 from dotenv import load_dotenv
+from loguru import logger
+from tqdm import tqdm
 
 load_dotenv()
 
@@ -11,8 +13,9 @@ load_dotenv()
 def removing(path):
     for root, dirs, files in os.walk(path):
         for dir in dirs:
-            if dir == '__pycache__':
-                os.system(f'rm -rf {os.path.join(root, dir)}')
+            if dir == "__pycache__":
+                pycache_path = os.path.join(root, dir)
+                subprocess.run(["rm", "-rf", pycache_path], check=False)
 
 
 def format_size_adaptive(value):
@@ -25,103 +28,125 @@ def format_size_adaptive(value):
 
 
 def format_datetime_into_isoformat(date_time: datetime.datetime) -> str:
-    return date_time.replace(tzinfo=datetime.timezone.utc).isoformat().replace(
-        "+00:00", "Z")
+    return (
+        date_time.replace(tzinfo=datetime.timezone.utc)
+        .isoformat()
+        .replace("+00:00", "Z")
+    )
 
 
 class OssClient:
-    def __init__(self, oss_access_key_id, oss_access_key_secret, oss_endpoint, oss_bucket_name):
+    def __init__(
+        self, oss_access_key_id, oss_access_key_secret, oss_endpoint, oss_bucket_name
+    ):
         self.bucket_name = oss_bucket_name
-        self.auth = oss2.Auth(os.getenv("OSS_ACCESS_KEY_ID", oss_access_key_id),
-                              os.getenv("OSS_ACCESS_KEY_SECRET", oss_access_key_secret))
+        self.auth = oss2.Auth(
+            os.getenv("OSS_ACCESS_KEY_ID", oss_access_key_id),
+            os.getenv("OSS_ACCESS_KEY_SECRET", oss_access_key_secret),
+        )
         self.endpoint = os.getenv("OSS_ENDPOINT", oss_endpoint)
-        self.bucket = oss2.Bucket(self.auth, self.endpoint, os.getenv("OSS_BUCKET_NAME", oss_bucket_name))
+        self.bucket = oss2.Bucket(
+            self.auth, self.endpoint, os.getenv("OSS_BUCKET_NAME", oss_bucket_name)
+        )
 
     # Upload a file
     # Usage: ossBucket.put_object_from_file("my-object-key", "path/to/local/file.txt")
     def put_object_from_file(self, object_name, file_path, progress_callback=None):
-        self.bucket.put_object_from_file(object_name, file_path, progress_callback=progress_callback)
+        self.bucket.put_object_from_file(
+            object_name, file_path, progress_callback=progress_callback
+        )
 
     # Download a file
     # Usage: ossBucket.get_object_to_file("my-object-key", "path/to/local/output-file.txt")
     def get_object_to_file(self, object_name, file_path, progress_callback=None):
         try:
-            self.bucket.get_object_to_file(object_name, file_path, progress_callback=progress_callback)
-        except oss2.exceptions.NoSuchKey as e:
+            self.bucket.get_object_to_file(
+                object_name, file_path, progress_callback=progress_callback
+            )
+        except oss2.exceptions.NoSuchKey:
             raise
 
             # Upload a folder
 
     # Usage: ossBucket.put_object_from_folder("my-object-folder", "path/to/local/folder")
-    def put_pdf_word_from_folder(self, object_folder_name, local_folder_path, progress_callback=None):
+    def put_pdf_word_from_folder(
+        self, object_folder_name, local_folder_path, progress_callback=None
+    ):
         for root, dirs, files in os.walk(local_folder_path):
             for file in tqdm(files, desc=root):
-                if file.endswith('.pdf') or file.endswith('.word'):
+                if file.endswith(".pdf") or file.endswith(".word"):
                     file_path = os.path.join(root, file)
                     object_name = os.path.join(
-                        object_folder_name, file_path[len(local_folder_path) + 1:])
-                    self.bucket.put_object_from_file(object_name, file_path, progress_callback=progress_callback)
+                        object_folder_name, file_path[len(local_folder_path) + 1 :]
+                    )
+                    self.bucket.put_object_from_file(
+                        object_name, file_path, progress_callback=progress_callback
+                    )
                     # logger.info("object name: {}, file path: {}".format(
                     #     object_name, file_path))
 
     # Upload a folder
     # Usage: ossBucket.put_object_from_folder("my-object-folder", "path/to/local/folder")
-    def put_object_from_folder(self, object_folder_name, local_folder_path, progress_callback=None):
+    def put_object_from_folder(
+        self, object_folder_name, local_folder_path, progress_callback=None
+    ):
         for root, dirs, files in os.walk(local_folder_path):
             for file in tqdm(files, desc=root):
                 file_path = os.path.join(root, file)
                 object_name = os.path.join(
-                    object_folder_name, file_path[len(local_folder_path) + 1:])
-                self.bucket.put_object_from_file(object_name, file_path, progress_callback=progress_callback)
-                logger.info("object name: {}, file path: {}".format(
-                    object_name, file_path))
+                    object_folder_name, file_path[len(local_folder_path) + 1 :]
+                )
+                self.bucket.put_object_from_file(
+                    object_name, file_path, progress_callback=progress_callback
+                )
+                logger.info(
+                    "object name: {}, file path: {}".format(object_name, file_path)
+                )
 
     # Download a folder
     # Usage: ossBucket.get_object_to_folder("my-object-folder", "path/to/local/output-folder")
-    def get_object_to_folder(self,
-                             object_folder_name,
-                             local_folder_path,
-                             progress_callback=None):
+    def get_object_to_folder(
+        self, object_folder_name, local_folder_path, progress_callback=None
+    ):
         os.makedirs(local_folder_path, exist_ok=True)
         for obj in oss2.ObjectIterator(self.bucket, prefix=object_folder_name):
-            file_path = os.path.join(local_folder_path,
-                                     obj.key[len(object_folder_name) + 1:])
-            self.bucket.get_object_to_file(obj.key,
-                                           file_path,
-                                           progress_callback=progress_callback)
+            file_path = os.path.join(
+                local_folder_path, obj.key[len(object_folder_name) + 1 :]
+            )
+            self.bucket.get_object_to_file(
+                obj.key, file_path, progress_callback=progress_callback
+            )
 
     # Get all objects in the bucket
     # Usage: ossBucket.get_all_objects_in_bucket()
     def get_all_objects_in_bucket(self, prefix=None, delimiter=None):
-        for obj in oss2.ObjectIterator(self.bucket,
-                                       prefix=prefix,
-                                       delimiter=delimiter):
+        for obj in oss2.ObjectIterator(self.bucket, prefix=prefix, delimiter=delimiter):
             if obj.is_prefix():  # obj is folder
                 logger.info("directory key: {}".format(obj.key))
             else:  # obj is file
                 logger.info(
-                    "file key: {}, object last modified: {}, object size: {}".
-                        format(
+                    "file key: {}, object last modified: {}, object size: {}".format(
                         obj.key,
                         format_datetime_into_isoformat(
-                            datetime.datetime.fromtimestamp(
-                                obj.last_modified)),
-                        format_size_adaptive(obj.size)))
+                            datetime.datetime.fromtimestamp(obj.last_modified)
+                        ),
+                        format_size_adaptive(obj.size),
+                    )
+                )
 
     def get_objects_in_folders(self, prefix: str):
         all_keys = []
-        for obj in oss2.ObjectIterator(self.bucket,
-                                       prefix=prefix):
+        for obj in oss2.ObjectIterator(self.bucket, prefix=prefix):
             if obj.is_prefix():  # obj is folder
                 pass
             else:  # obj is file
-                if obj.key.endswith('/'):
+                if obj.key.endswith("/"):
                     continue
                 all_keys.append(obj.key)
         return all_keys
 
-    def delete_object(self, object_name='test'):
-        if object_name is None or object_name == '':
+    def delete_object(self, object_name="test"):
+        if object_name is None or object_name == "":
             raise Exception(
                 "Danger! object name is None or '' Will delete all objects in bucket!"
             )
@@ -129,8 +154,8 @@ class OssClient:
 
     # Delete a folder
     # Usage: ossBucket.delete_object_folder("my-object-folder")
-    def delete_object_folder(self, object_folder_name='test'):
-        if object_folder_name is None or object_folder_name == '':
+    def delete_object_folder(self, object_folder_name="test"):
+        if object_folder_name is None or object_folder_name == "":
             raise Exception(
                 "Danger! object name is None or '' Will delete all objects in bucket!"
             )
@@ -138,20 +163,29 @@ class OssClient:
             self.bucket.delete_object(obj.key)
             logger.info("delete object key: {}".format(obj.key))
 
-    def get_oss_url(self, object_name, url_expires_time, aliyun_oss_url_prefix, csnt_url_prefix):
+    def get_oss_url(
+        self, object_name, url_expires_time, aliyun_oss_url_prefix, csnt_url_prefix
+    ):
         oss_prefix = "oss://" + os.getenv("OSS_BUCKET_NAME", self.bucket_name) + "/"
         if object_name.__contains__(oss_prefix):
             object_name = object_name.replace(oss_prefix, "")
         aliyun_url = self.bucket.sign_url(
-            "GET", object_name, int(os.getenv("URL_EXPIRES_TIME", url_expires_time)), slash_safe=True
+            "GET",
+            object_name,
+            int(os.getenv("URL_EXPIRES_TIME", url_expires_time)),
+            slash_safe=True,
         )
         csnt_url = aliyun_url.replace(
-            os.getenv("ALIYUN_OSS_URL_PREFIX", aliyun_oss_url_prefix), os.getenv("CSNT_URL_PREFIX", csnt_url_prefix)
+            os.getenv("ALIYUN_OSS_URL_PREFIX", aliyun_oss_url_prefix),
+            os.getenv("CSNT_URL_PREFIX", csnt_url_prefix),
         )
         return csnt_url
 
     def get_default_oss_url(self, object_name: str, url_expires_time):
         aliyun_url = self.bucket.sign_url(
-            "GET", object_name, int(os.getenv("url_expires_time", url_expires_time)), slash_safe=True
+            "GET",
+            object_name,
+            int(os.getenv("url_expires_time", url_expires_time)),
+            slash_safe=True,
         )
         return aliyun_url

datamax/parser/__init__.py

@@ -1,4 +1,4 @@
-from .core import DataMaxParser
+from .core import DataMax
 import logging
 logger = logging.getLogger()
 logger.addHandler(logging.NullHandler())

datamax/parser/base.py

@@ -58,12 +58,12 @@ class BaseLife:
     tk_client = DashScopeClient()
 
     @staticmethod
-    def generate_lifecycle(source_file, token_count, domain, life_type, usage_purpose) -> LifeCycle:
+    def generate_lifecycle(source_file, domain, life_type, usage_purpose) -> LifeCycle:
         update_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         life_type = [life_type]
         storage = os.stat(source_file)
         life_metadata = {
-            "token_count": token_count,  # Token count of the text
+            # "token_count": token_count,  # Token count of the text
             "storage_size": storage.st_size,  # Storage size in bytes
             "source_file": source_file,  # Source file
             "domain": domain,  # Domain

datamax/parser/core.py

@@ -1,24 +1,49 @@
 import os
 import importlib
+from typing import List, Union, Dict
+from openai import OpenAI
+from datamax.utils import data_cleaner
+from datamax.utils.qa_generator import generatr_qa_pairs
+
+
+class ModelInvoker:
+    def __init__(self):
+        self.client = None
+
+    def invoke_model(self, api_key, base_url, model_name, messages):
+        self.client = OpenAI(
+            api_key=api_key,
+            base_url=base_url,
+        )
+
+        completion = self.client.chat.completions.create(
+            model=model_name,
+            messages=messages,
+        )
+        json_data = completion.model_dump()
+        return json_data.get("choices")[0].get("message").get("content", "")
 
 
 class ParserFactory:
     @staticmethod
-    def create_parser(file_path: str, use_ocr: bool = False, use_gpu: bool = False, gpu_id: int = 6,
-                      to_markdown: bool = False):
+    def create_parser(
+            file_path: str,
+            use_mineru: bool = False,
+            to_markdown: bool = False,
+            timeout: int = 1200
+    ):
         """
         Create a parser instance based on the file extension.
-
         :param file_path: The path to the file to be parsed.
-        :param use_ocr: Flag to indicate whether OCR should be used.
-        :param use_gpu: Flag to indicate whether GPU should be used.
-        :param gpu_id: The ID of the GPU to use.
         :param to_markdown: Flag to indicate whether the output should be in Markdown format.
                     (only supported files in .doc or .docx format)
+        :param use_mineru: Flag to indicate whether MinerU should be used. (only supported files in .pdf format)
+        :param timeout: Timeout for the request .(only supported files in .xlsx format)
         :return: An instance of the parser class corresponding to the file extension.
         """
         file_extension = os.path.splitext(file_path)[1].lower()
         parser_class_name = {
+            '.md': 'MarkdownParser',
             '.docx': 'DocxParser',
             '.doc': 'DocParser',
             '.epub': 'EpubParser',
@@ -28,13 +53,17 @@ class ParserFactory:
             '.ppt': 'PPtParser',
             '.pdf': 'PdfParser',
             '.jpg': 'ImageParser',
-            '.png': 'ImageParser'
+            '.jpeg': 'ImageParser',
+            '.png': 'ImageParser',
+            '.webp': 'ImageParser',
+            '.xlsx': 'XlsxParser',
+            '.xls': 'XlsParser'
         }.get(file_extension)
 
         if not parser_class_name:
             return None
 
-        if file_extension == '.jpg' or file_extension == '.png':
+        if file_extension in ['.jpg', 'jpeg', '.png', '.webp']:
             module_name = f'datamax.parser.image_parser'
         else:
             # Dynamically determine the module name based on the file extension
@@ -47,33 +76,55 @@ class ParserFactory:
 
             # Special handling for PdfParser arguments
             if parser_class_name == 'PdfParser':
-                return parser_class(file_path, use_ocr, use_gpu, gpu_id)
+                return parser_class(
+                    file_path=file_path,
+                    use_mineru=use_mineru,
+                )
             elif parser_class_name == 'DocxParser' or parser_class_name == 'DocParser':
-                return parser_class(file_path, to_markdown)
+                return parser_class(
+                    file_path=file_path, to_markdown=to_markdown
+                )
+            elif parser_class_name == 'XlsxParser':
+                return parser_class(
+                    file_path=file_path,
+                    timeout=timeout
+                )
             else:
-                return parser_class(file_path)
+                return parser_class(
+                    file_path=file_path
+                )
 
         except (ImportError, AttributeError) as e:
             raise e
 
 
-class DataMaxParser:
-    def __init__(self, file_path, use_ocr: bool = False, use_gpu: bool = False, gpu_id: int = 6,
-                 to_markdown: bool = False):
+class DataMax:
+    def __init__(self,
+                 file_path: Union[str, list] = '',
+                 use_mineru: bool = False,
+                 to_markdown: bool = False,
+                 timeout: int = 1200
+                 ):
         """
         Initialize the DataMaxParser with file path and parsing options.
 
+        # <Abandon>
+        # :param use_paddle_ocr: Flag to indicate whether PaddleOCR should be used.
+        # :param use_paddle_gpu: Flag to indicate whether PaddleOCR-GPU should be used.
+        # :param use_got_ocr: Flag to indicate whether GOT-OCR should be used.
+        # :param got_weights_path: GOT-OCR Weights Path.
+        # :param gpu_id: The ID of the GPU to use.
+
         :param file_path: The path to the file or directory to be parsed.
-        :param use_ocr: Flag to indicate whether OCR should be used.
-        :param use_gpu: Flag to indicate whether GPU should be used.
-        :param gpu_id: The ID of the GPU to use.
+        :param use_mineru: Flag to indicate whether MinerU should be used.
         :param to_markdown: Flag to indicate whether the output should be in Markdown format.
         """
         self.file_path = file_path
-        self.use_ocr = use_ocr
-        self.use_gpu = use_gpu
-        self.gpu_id = gpu_id
+        self.use_mineru = use_mineru
         self.to_markdown = to_markdown
+        self.parsed_data = None
+        self.model_invoker = ModelInvoker()
+        self.timeout = timeout
 
     def get_data(self):
         """
@@ -83,19 +134,136 @@ class DataMaxParser:
         """
         try:
             if isinstance(self.file_path, list):
-                data = [self._parse_file(f) for f in self.file_path]
-                return data
+                parsed_data = [self._parse_file(f) for f in self.file_path]
+                self.parsed_data = parsed_data
+                return parsed_data
 
             elif isinstance(self.file_path, str) and os.path.isfile(self.file_path):
-                return self._parse_file(self.file_path)
+                parsed_data = self._parse_file(self.file_path)
+                self.parsed_data = parsed_data
+                return parsed_data
 
             elif isinstance(self.file_path, str) and os.path.isdir(self.file_path):
                 file_list = [os.path.join(self.file_path, file) for file in os.listdir(self.file_path)]
-                data = [self._parse_file(f) for f in file_list if os.path.isfile(f)]
-                return data
+                parsed_data = [self._parse_file(f) for f in file_list if os.path.isfile(f)]
+                self.parsed_data = parsed_data
+                return parsed_data
+            else:
+                raise ValueError("Invalid file path.")
+
         except Exception as e:
             raise e
 
+    def clean_data(self, method_list: List[str], text: str = None):
+        """
+        Clean data
+
+        methods include AbnormalCleaner， TextFilter， PrivacyDesensitization which is 1 2 3
+
+        :return:
+        """
+        if text:
+            cleaned_text = text
+        elif self.parsed_data:
+            cleaned_text = self.parsed_data.get('content')
+        else:
+            raise ValueError("No data to clean.")
+
+        for method in method_list:
+            if method == 'abnormal':
+                cleaned_text = data_cleaner.AbnormalCleaner(cleaned_text).to_clean().get("text")
+            elif method == 'filter':
+                cleaned_text = data_cleaner.TextFilter(cleaned_text).to_filter()
+                cleaned_text = cleaned_text.get("text") if cleaned_text else ''
+            elif method == 'private':
+                cleaned_text = data_cleaner.PrivacyDesensitization(cleaned_text).to_private().get("text")
+
+        if self.parsed_data:
+            origin_dict = self.parsed_data
+            origin_dict['content'] = cleaned_text
+            self.parsed_data = None
+            return origin_dict
+        else:
+            return cleaned_text
+
+    def get_pre_label(self,
+                      api_key: str,
+                      base_url: str,
+                      model_name: str,
+                      chunk_size: int = 500,
+                      chunk_overlap: int = 100,
+                      question_number: int = 5,
+                      max_workers: int = 5,
+                      messages: List[Dict[str, str]] = None):
+        return generatr_qa_pairs(
+            api_key=api_key,
+            base_url=base_url,
+            model_name=model_name,
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+            question_number=question_number,
+            max_workers=max_workers,
+            message=messages,
+            file_path=self.file_path
+        )
+
+    ## <Abandon>
+    # def enhance_with_model(self, api_key: str, base_url: str, model_name: str, iteration: int = 1,
+    #                        messages: List[Dict[str, str]] = None):
+    #     """
+    #     Enhance the parsed content using a large language model.
+    #
+    #     :param api_key: API key for the large model service.
+    #     :param base_url: Base URL for the large model service.
+    #     :param model_name: Name of the model to use.
+    #     :param iteration: Number of iterations
+    #     :param messages: Custom messages list [{"role": "system", "content": "..."}, ...]
+    #     :return: Enhanced text.
+    #     """
+    #     if not messages:
+    #         # If no custom message is provided, the default message structure is used, but only if there is parsed data
+    #         if self.parsed_data:
+    #             system_prompt = get_system_prompt(self.parsed_data)
+    #             default_message_user = {"role": "user", "content": "按照json格式给出问答对"}
+    #             messages = [
+    #                 {"role": "system", "content": system_prompt},
+    #                 default_message_user
+    #             ]
+    #         else:
+    #             raise ValueError("No data to enhance and no custom messages provided.")
+    #     try:
+    #         if isinstance(iteration, int) and iteration >= 1:
+    #             results = []
+    #             current_messages = messages.copy()  # Avoid modifying the original message during iteration
+    #
+    #             for _ in range(iteration):
+    #                 enhanced_text = self.model_invoker.invoke_model(
+    #                     api_key=api_key,
+    #                     base_url=base_url,
+    #                     model_name=model_name,
+    #                     messages=current_messages
+    #                 )
+    #
+    #                 # Append the generated content to the conversation history in multiple iterations
+    #                 if iteration > 1:
+    #                     current_messages.append({"role": "assistant", "content": enhanced_text})
+    #                     current_messages.append(
+    #                         {"role": "user", "content": "请继续生成, 生成要求不变, 结果是jsonlist, 且长度不超过5"})
+    #
+    #                 # If there is parsed data, update the contents and return a copy of the original dictionary; Otherwise, return the enhanced text directly
+    #                 if self.parsed_data:
+    #                     origin_dict = self.parsed_data.copy()
+    #                     origin_dict['content'] = enhanced_text
+    #                     results.append(origin_dict)
+    #                 else:
+    #                     results.append(enhanced_text)
+    #
+    #             return results if iteration > 1 else results[0]
+    #         else:
+    #             raise ValueError("Invalid iteration parameter.")
+    #     except Exception as e:
+    #         raise Exception(f"An error occurred while enhancing with the model: {e}")
+
     def _parse_file(self, file_path):
         """
         Create a parser instance using ParserFactory and parse the file.
@@ -103,12 +271,18 @@ class DataMaxParser:
         :param file_path: The path to the file to be parsed.
         :return: The parsed data.
         """
-        parser = ParserFactory.create_parser(file_path, self.use_ocr, self.use_gpu, self.gpu_id, self.to_markdown)
-        if parser:
-            return parser.parse(file_path)
+        try:
+            parser = ParserFactory.create_parser(
+                use_mineru=self.use_mineru,
+                file_path=file_path,
+                to_markdown=self.to_markdown,
+                timeout=self.timeout
+            )
+            if parser:
+                return parser.parse(file_path=file_path)
+        except Exception as e:
+            raise e
 
 
 if __name__ == '__main__':
-    data = DataMaxParser(file_path=r"C:\Users\cykro\Desktop\数据工厂.pdf", use_ocr=True)
-    data = data.get_data()
-    print(data)
+    pass

datamax/parser/doc_parser.py

@@ -9,7 +9,6 @@ from typing import Union
 from docx import Document
 from datamax.parser.base import BaseLife
 from datamax.parser.base import MarkdownOutputVo
-from datamax.utils import clean_original_text
 
 
 class DocParser(BaseLife):
@@ -68,10 +67,8 @@ class DocParser(BaseLife):
                 mk_content = self.read_docx_file(doc_path=file_path, to_mk=True)
             else:
                 content = self.read_docx_file(doc_path=file_path, to_mk=False)
-                clean_text = clean_original_text(content)
-                mk_content = clean_text.get('text', '')
-            token_count = self.tk_client.get_tokenizer(content=mk_content)
-            lifecycle = self.generate_lifecycle(source_file=file_path, token_count=token_count, domain="Technology",
+                mk_content = content
+            lifecycle = self.generate_lifecycle(source_file=file_path, domain="Technology",
                                                 usage_purpose="Documentation", life_type="LLM_ORIGIN")
             output_vo = MarkdownOutputVo(title, mk_content)
             output_vo.add_lifecycle(lifecycle)

datamax/parser/docx_parser.py

@@ -4,7 +4,6 @@ from docx import Document
 from typing import Union
 from datamax.parser.base import BaseLife
 from datamax.parser.base import MarkdownOutputVo
-from datamax.utils import clean_original_text
 
 
 class DocxParser(BaseLife):
@@ -34,13 +33,11 @@ class DocxParser(BaseLife):
                 mk_content = open(output_md_dir, 'r', encoding='utf-8').read()
             else:
                 content = self.read_docx_file(file_path=file_path)
-                clean_text = clean_original_text(content)
-                mk_content = clean_text.get('text', '')
-            token_count = self.tk_client.get_tokenizer(content=mk_content)
-            lifecycle = self.generate_lifecycle(source_file=file_path, token_count=token_count, domain="Technology",
+                mk_content = content
+            lifecycle = self.generate_lifecycle(source_file=file_path, domain="Technology",
                                                 usage_purpose="Documentation", life_type="LLM_ORIGIN")
             output_vo = MarkdownOutputVo(title, mk_content)
             output_vo.add_lifecycle(lifecycle)
             return output_vo.to_dict()
         except Exception as e:
-            raise e
+            raise e

datamax/parser/epub_parser.py

@@ -4,7 +4,6 @@ from bs4 import BeautifulSoup
 from ebooklib import epub
 from datamax.parser.base import BaseLife
 from datamax.parser.base import MarkdownOutputVo
-from datamax.utils import clean_original_text
 
 
 class EpubParser(BaseLife):
@@ -32,10 +31,8 @@ class EpubParser(BaseLife):
         try:
             title = self.get_file_extension(file_path)
             content = self.read_epub_file(file_path=file_path)
-            clean_text = clean_original_text(content)
-            mk_content = clean_text.get('text', '')
-            token_count = self.tk_client.get_tokenizer(content=mk_content)
-            lifecycle = self.generate_lifecycle(source_file=file_path, token_count=token_count, domain="Technology",
+            mk_content = content
+            lifecycle = self.generate_lifecycle(source_file=file_path, domain="Technology",
                                                 usage_purpose="Documentation", life_type="LLM_ORIGIN")
             output_vo = MarkdownOutputVo(title, mk_content)
             output_vo.add_lifecycle(lifecycle)

datamax/parser/html_parser.py

@@ -7,7 +7,6 @@ sys.path.insert(0, str(ROOT_DIR))
 from datamax.parser.base import BaseLife
 from datamax.parser.base import MarkdownOutputVo
 from bs4 import BeautifulSoup
-from datamax.utils import clean_original_text
 
 
 class HtmlParser(BaseLife):
@@ -29,10 +28,8 @@ class HtmlParser(BaseLife):
         try:
             title = self.get_file_extension(file_path)
             content = self.read_html_file(file_path=file_path)
-            clean_text = clean_original_text(content)
-            mk_content = clean_text.get('text')
-            token_count = self.tk_client.get_tokenizer(content=mk_content)
-            lifecycle = self.generate_lifecycle(source_file=file_path, token_count=token_count, domain="Technology",
+            mk_content = content
+            lifecycle = self.generate_lifecycle(source_file=file_path, domain="Technology",
                                                 usage_purpose="Documentation", life_type="LLM_ORIGIN")
             output_vo = MarkdownOutputVo(title, mk_content)
             output_vo.add_lifecycle(lifecycle)