pydatamax 0.1.13__py3-none-any.whl → 0.1.15__py3-none-any.whl

datamax/parser/epub_parser.py

@@ -1,41 +1,41 @@
-import ebooklib
-from typing import Union
-from bs4 import BeautifulSoup
-from ebooklib import epub
-from datamax.parser.base import BaseLife
-from datamax.parser.base import MarkdownOutputVo
-
-
-class EpubParser(BaseLife):
-    def __init__(self, file_path: Union[str, list]):
-        super().__init__()
-        self.file_path = file_path
-
-    @staticmethod
-    def read_epub_file(file_path: str) -> str:
-        try:
-            book = epub.read_epub(file_path)
-            content = ""
-            for item in book.get_items():
-                if item.get_type() == ebooklib.ITEM_DOCUMENT:
-                    chapter_content = item.get_content().decode('utf-8')
-                    soup = BeautifulSoup(chapter_content, 'html.parser')
-                    text = soup.get_text()
-                    text = text.replace('\u3000', ' ')
-                    content += text
-            return content
-        except Exception as e:
-            raise e
-
-    def parse(self, file_path: str) -> MarkdownOutputVo:
-        try:
-            title = self.get_file_extension(file_path)
-            content = self.read_epub_file(file_path=file_path)
-            mk_content = content
-            lifecycle = self.generate_lifecycle(source_file=file_path, domain="Technology",
-                                                usage_purpose="Documentation", life_type="LLM_ORIGIN")
-            output_vo = MarkdownOutputVo(title, mk_content)
-            output_vo.add_lifecycle(lifecycle)
-            return output_vo.to_dict()
-        except Exception as e:
-            raise e
+import ebooklib
+from typing import Union
+from bs4 import BeautifulSoup
+from ebooklib import epub
+from datamax.parser.base import BaseLife
+from datamax.parser.base import MarkdownOutputVo
+import os
+
+class EpubParser(BaseLife):
+    def __init__(self, file_path: Union[str, list]):
+        super().__init__()
+        self.file_path = file_path
+
+    @staticmethod
+    def read_epub_file(file_path: str) -> str:
+        try:
+            book = epub.read_epub(file_path)
+            content = ""
+            for item in book.get_items():
+                if item.get_type() == ebooklib.ITEM_DOCUMENT:
+                    chapter_content = item.get_content().decode('utf-8')
+                    soup = BeautifulSoup(chapter_content, 'html.parser')
+                    text = soup.get_text()
+                    text = text.replace('\u3000', ' ')
+                    content += text
+            return content
+        except Exception as e:
+            raise e
+
+    def parse(self, file_path: str) -> MarkdownOutputVo:
+        try:
+            title = os.path.splitext(os.path.basename(file_path))[0]
+            content = self.read_epub_file(file_path=file_path)
+            mk_content = content
+            lifecycle = self.generate_lifecycle(source_file=file_path, domain="Technology",
+                                                usage_purpose="Documentation", life_type="LLM_ORIGIN")
+            output_vo = MarkdownOutputVo(title, mk_content)
+            output_vo.add_lifecycle(lifecycle)
+            return output_vo.to_dict()
+        except Exception as e:
+            raise e

datamax/parser/html_parser.py

@@ -1,38 +1,38 @@
-from typing import Union
-import pathlib
-import sys
-
-ROOT_DIR: pathlib.Path = pathlib.Path(__file__).parent.parent.parent.resolve()
-sys.path.insert(0, str(ROOT_DIR))
-from datamax.parser.base import BaseLife
-from datamax.parser.base import MarkdownOutputVo
-from bs4 import BeautifulSoup
-
-
-class HtmlParser(BaseLife):
-    def __init__(self, file_path: Union[str, list]):
-        super().__init__()
-        self.file_path = file_path
-
-    @staticmethod
-    def read_html_file(file_path: str) -> str:
-        try:
-            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
-                data = f.read()
-                soup = BeautifulSoup(data, 'html.parser')
-                return soup.get_text(separator='\n', strip=True)
-        except Exception:
-            raise
-
-    def parse(self, file_path: str) -> MarkdownOutputVo:
-        try:
-            title = self.get_file_extension(file_path)
-            content = self.read_html_file(file_path=file_path)
-            mk_content = content
-            lifecycle = self.generate_lifecycle(source_file=file_path, domain="Technology",
-                                                usage_purpose="Documentation", life_type="LLM_ORIGIN")
-            output_vo = MarkdownOutputVo(title, mk_content)
-            output_vo.add_lifecycle(lifecycle)
-            return output_vo.to_dict()
-        except Exception:
+from typing import Union
+import pathlib
+import sys
+
+ROOT_DIR: pathlib.Path = pathlib.Path(__file__).parent.parent.parent.resolve()
+sys.path.insert(0, str(ROOT_DIR))
+from datamax.parser.base import BaseLife
+from datamax.parser.base import MarkdownOutputVo
+from bs4 import BeautifulSoup
+import os
+
+class HtmlParser(BaseLife):
+    def __init__(self, file_path: Union[str, list]):
+        super().__init__()
+        self.file_path = file_path
+
+    @staticmethod
+    def read_html_file(file_path: str) -> str:
+        try:
+            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                data = f.read()
+                soup = BeautifulSoup(data, 'html.parser')
+                return soup.get_text(separator='\n', strip=True)
+        except Exception:
+            raise
+
+    def parse(self, file_path: str) -> MarkdownOutputVo:
+        try:
+            title = os.path.splitext(os.path.basename(file_path))[0]
+            content = self.read_html_file(file_path=file_path)
+            mk_content = content
+            lifecycle = self.generate_lifecycle(source_file=file_path, domain="Technology",
+                                                usage_purpose="Documentation", life_type="LLM_ORIGIN")
+            output_vo = MarkdownOutputVo(title, mk_content)
+            output_vo.add_lifecycle(lifecycle)
+            return output_vo.to_dict()
+        except Exception:
             raise

datamax/parser/image_parser.py

@@ -1,34 +1,34 @@
-import os
-import pathlib
-import sys
-from datamax.utils import setup_environment
-
-setup_environment(use_gpu=True)
-os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
-from datamax.parser.base import MarkdownOutputVo
-
-ROOT_DIR: pathlib.Path = pathlib.Path(__file__).parent.parent.parent.resolve()
-sys.path.insert(0, str(ROOT_DIR))
-from datamax.parser.base import BaseLife
-from datamax.parser.pdf_parser import PdfParser
-from PIL import Image
-
-class ImageParser(BaseLife):
-    def __init__(self,file_path: str):
-        super().__init__()
-        self.file_path = file_path
-
-    def parse(self, file_path: str) -> MarkdownOutputVo:
-        try:
-            title = self.get_file_extension(file_path)
-            output_pdf_path = f'{os.path.basename(file_path).strip(title)}.pdf'
-            image = Image.open(file_path)
-            image.save(output_pdf_path, 'PDF', resolution=100.0)
-            pdf_parser = PdfParser(output_pdf_path, use_mineru=True)
-            output_vo = pdf_parser.parse(output_pdf_path)
-            if os.path.exists(output_pdf_path):
-                # shutil.rmtree(f'uploaded_files/markdown')
-                os.remove(output_pdf_path)
-            return output_vo
-        except Exception as e:
-            raise e
+import os
+import pathlib
+import sys
+from datamax.utils import setup_environment
+
+setup_environment(use_gpu=True)
+os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
+from datamax.parser.base import MarkdownOutputVo
+
+ROOT_DIR: pathlib.Path = pathlib.Path(__file__).parent.parent.parent.resolve()
+sys.path.insert(0, str(ROOT_DIR))
+from datamax.parser.base import BaseLife
+from datamax.parser.pdf_parser import PdfParser
+from PIL import Image
+
+class ImageParser(BaseLife):
+    def __init__(self,file_path: str):
+        super().__init__()
+        self.file_path = file_path
+
+    def parse(self, file_path: str) -> MarkdownOutputVo:
+        try:
+            title = os.path.splitext(os.path.basename(file_path))[0]
+            output_pdf_path = f'{os.path.basename(file_path).strip(title)}.pdf'
+            image = Image.open(file_path)
+            image.save(output_pdf_path, 'PDF', resolution=100.0)
+            pdf_parser = PdfParser(output_pdf_path, use_mineru=True)
+            output_vo = pdf_parser.parse(output_pdf_path)
+            if os.path.exists(output_pdf_path):
+                # shutil.rmtree(f'uploaded_files/markdown')
+                os.remove(output_pdf_path)
+            return output_vo
+        except Exception as e:
+            raise e

datamax/parser/json_parser.py

@@ -1,10 +1,32 @@
-from datamax.parser.base import MarkdownOutputVo
-
-
-class Parser:
-
-    def __init__(self, file_path):
-        self.file_path = file_path
-
-    def parse(self) -> MarkdownOutputVo:
-        pass
+import json
+
+from datamax.parser.base import BaseLife, MarkdownOutputVo
+
+
+class JsonParser(BaseLife):
+
+    def __init__(self, file_path):
+        super().__init__()
+        self.file_path = file_path
+
+    @staticmethod
+    def read_json_file(file_path: str) -> str:
+        """Read and pretty print a JSON file."""
+        with open(file_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        return json.dumps(data, indent=2, ensure_ascii=False)
+
+    def parse(self, file_path: str) -> MarkdownOutputVo:
+        try:
+            content = self.read_json_file(file_path)
+            lifecycle = self.generate_lifecycle(
+                source_file=file_path,
+                domain="Technology",
+                usage_purpose="Documentation",
+                life_type="LLM_ORIGIN",
+            )
+            output_vo = MarkdownOutputVo(self.get_file_extension(file_path), content)
+            output_vo.add_lifecycle(lifecycle)
+            return output_vo.to_dict()
+        except Exception as e:
+            raise e

datamax/parser/md_parser.py

@@ -1,73 +1,73 @@
-import pathlib
-import sys
-from typing import Union
-
-ROOT_DIR: pathlib.Path = pathlib.Path(__file__).parent.parent.parent.resolve()
-sys.path.insert(0, str(ROOT_DIR))
-from datamax.parser.base import BaseLife
-from datamax.parser.base import MarkdownOutputVo
-from loguru import logger
-
-
-class MarkdownParser(BaseLife):
-    """
-    Parser for Markdown files that follows the same pattern as PdfParser.
-    Handles .md and .markdown file extensions.
-    """
-
-    def __init__(self,
-                 file_path: Union[str, list],
-                 ):
-        super().__init__()
-        self.file_path = file_path
-
-    @staticmethod
-    def read_markdown_file(file_path: str) -> str:
-        """
-        Reads the content of a markdown file.
-
-        Args:
-            file_path: Path to the markdown file
-
-        Returns:
-            str: Content of the markdown file
-        """
-        try:
-            with open(file_path, 'r', encoding='utf-8') as f:
-                return f.read()
-        except Exception as e:
-            logger.error(f"Error reading markdown file {file_path}: {e}")
-            raise
-
-    def parse(self, file_path: str) -> MarkdownOutputVo:
-        """
-        Parses a markdown file and returns a MarkdownOutputVo.
-
-        Args:
-            file_path: Path to the markdown file
-
-        Returns:
-            MarkdownOutputVo: Structured output containing the markdown content
-        """
-        try:
-            title = self.get_file_extension(file_path)
-
-            # Read markdown content
-            md_content = self.read_markdown_file(file_path)
-
-            # Generate lifecycle metadata
-            lifecycle = self.generate_lifecycle(
-                source_file=file_path,
-                domain="Technology",
-                usage_purpose="Documentation",
-                life_type="LLM_ORIGIN"
-            )
-
-            # Create and return output VO
-            output_vo = MarkdownOutputVo(title, md_content)
-            output_vo.add_lifecycle(lifecycle)
-            return output_vo.to_dict()
-
-        except Exception as e:
-            logger.error(f"Failed to parse markdown file {file_path}: {e}")
+import pathlib
+import sys
+from typing import Union
+
+ROOT_DIR: pathlib.Path = pathlib.Path(__file__).parent.parent.parent.resolve()
+sys.path.insert(0, str(ROOT_DIR))
+from datamax.parser.base import BaseLife
+from datamax.parser.base import MarkdownOutputVo
+from loguru import logger
+import os
+
+class MarkdownParser(BaseLife):
+    """
+    Parser for Markdown files that follows the same pattern as PdfParser.
+    Handles .md and .markdown file extensions.
+    """
+
+    def __init__(self,
+                 file_path: Union[str, list],
+                 ):
+        super().__init__()
+        self.file_path = file_path
+
+    @staticmethod
+    def read_markdown_file(file_path: str) -> str:
+        """
+        Reads the content of a markdown file.
+
+        Args:
+            file_path: Path to the markdown file
+
+        Returns:
+            str: Content of the markdown file
+        """
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                return f.read()
+        except Exception as e:
+            logger.error(f"Error reading markdown file {file_path}: {e}")
+            raise
+
+    def parse(self, file_path: str) -> MarkdownOutputVo:
+        """
+        Parses a markdown file and returns a MarkdownOutputVo.
+
+        Args:
+            file_path: Path to the markdown file
+
+        Returns:
+            MarkdownOutputVo: Structured output containing the markdown content
+        """
+        try:
+            title = os.path.splitext(os.path.basename(file_path))[0]
+
+            # Read markdown content
+            md_content = self.read_markdown_file(file_path)
+
+            # Generate lifecycle metadata
+            lifecycle = self.generate_lifecycle(
+                source_file=file_path,
+                domain="Technology",
+                usage_purpose="Documentation",
+                life_type="LLM_ORIGIN"
+            )
+
+            # Create and return output VO
+            output_vo = MarkdownOutputVo(title, md_content)
+            output_vo.add_lifecycle(lifecycle)
+            return output_vo.to_dict()
+
+        except Exception as e:
+            logger.error(f"Failed to parse markdown file {file_path}: {e}")
             raise

datamax/parser/pdf_parser.py

@@ -1,101 +1,101 @@
-import os
-import pathlib
-import sys
-import subprocess
-from typing import Union
-
-ROOT_DIR: pathlib.Path = pathlib.Path(__file__).parent.parent.parent.resolve()
-sys.path.insert(0, str(ROOT_DIR))
-from datamax.parser.base import BaseLife
-from datamax.parser.base import MarkdownOutputVo
-from langchain_community.document_loaders import PyMuPDFLoader
-from loguru import logger
-from datamax.utils.mineru_operator import pdf_processor
-
-
-class PdfParser(BaseLife):
-
-    def __init__(self,
-                 file_path: Union[str, list],
-                 use_mineru: bool = False,
-                 ):
-        super().__init__()
-
-        self.file_path = file_path
-        self.use_mineru = use_mineru
-
-    def mineru_process(self, input_pdf_filename, output_dir):
-        proc = None
-        try:
-            logger.info(f"mineru is working...\n input_pdf_filename: {input_pdf_filename} | output_dir: ./{output_dir}. plz waiting!")
-            command = ['magic-pdf', '-p', input_pdf_filename, '-o', output_dir]
-            proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
-            # 等待命令执行完成
-            stdout, stderr = proc.communicate()
-            # 检查命令是否成功执行
-            if proc.returncode != 0:
-                raise Exception(f"mineru failed with return code {proc.returncode}: {stderr.decode()}")
-
-            logger.info(f"Markdown saved in {output_dir}, input file is {input_pdf_filename}")
-
-        except Exception as e:
-            logger.error(f"Error: {e}")
-            if proc is not None:
-                proc.kill()
-                proc.wait()
-                logger.info("The process was terminated due to an error.")
-            raise  # Re-raise the exception to let the caller handle it
-
-        finally:
-            # 确保子进程已经结束
-            if proc is not None:
-                if proc.poll() is None:
-                    proc.kill()
-                    proc.wait()
-                    logger.info("The process was terminated due to timeout or completion.")
-
-    @staticmethod
-    def read_pdf_file(file_path) -> str:
-        try:
-            pdf_loader = PyMuPDFLoader(file_path)
-            pdf_documents = pdf_loader.load()
-            result_text = ''
-            for page in pdf_documents:
-                result_text += page.page_content
-            return result_text
-        except Exception as e:
-            raise e
-
-    def parse(self, file_path: str) -> MarkdownOutputVo:
-        try:
-            title = self.get_file_extension(file_path)
-
-            if self.use_mineru:
-                output_dir = 'uploaded_files'
-                output_folder_name = os.path.basename(file_path).replace(".pdf", "")
-                # output_mineru = f'{output_dir}/{output_folder_name}/auto/{output_folder_name}.md'
-                # if os.path.exists(output_mineru):
-                #     pass
-                # else:
-                    # self.mineru_process(input_pdf_filename=file_path, output_dir=output_dir)
-                # mk_content = open(output_mineru, 'r', encoding='utf-8').read()
-
-                # todo: 是否有必要跟api的默认保存路径保持一致
-                output_mineru = f'{output_dir}/markdown/{output_folder_name}.md'
-
-                if os.path.exists(output_mineru):
-                    mk_content = open(output_mineru, 'r', encoding='utf-8').read()
-                else:
-                    mk_content = pdf_processor.process_pdf(file_path)
-            else:
-                content = self.read_pdf_file(file_path=file_path)
-                mk_content = content
-
-            lifecycle = self.generate_lifecycle(source_file=file_path, domain="Technology",
-                                                usage_purpose="Documentation", life_type="LLM_ORIGIN")
-            output_vo = MarkdownOutputVo(title, mk_content)
-            output_vo.add_lifecycle(lifecycle)
-            return output_vo.to_dict()
-        except Exception:
-            raise
+import os
+import pathlib
+import sys
+import subprocess
+from typing import Union
+
+ROOT_DIR: pathlib.Path = pathlib.Path(__file__).parent.parent.parent.resolve()
+sys.path.insert(0, str(ROOT_DIR))
+from datamax.parser.base import BaseLife
+from datamax.parser.base import MarkdownOutputVo
+from langchain_community.document_loaders import PyMuPDFLoader
+from loguru import logger
+from datamax.utils.mineru_operator import pdf_processor
+import os
+
+class PdfParser(BaseLife):
+
+    def __init__(self,
+                 file_path: Union[str, list],
+                 use_mineru: bool = False,
+                 ):
+        super().__init__()
+
+        self.file_path = file_path
+        self.use_mineru = use_mineru
+
+    def mineru_process(self, input_pdf_filename, output_dir):
+        proc = None
+        try:
+            logger.info(f"mineru is working...\n input_pdf_filename: {input_pdf_filename} | output_dir: ./{output_dir}. plz waiting!")
+            command = ['magic-pdf', '-p', input_pdf_filename, '-o', output_dir]
+            proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+            # 等待命令执行完成
+            stdout, stderr = proc.communicate()
+            # 检查命令是否成功执行
+            if proc.returncode != 0:
+                raise Exception(f"mineru failed with return code {proc.returncode}: {stderr.decode()}")
+
+            logger.info(f"Markdown saved in {output_dir}, input file is {input_pdf_filename}")
+
+        except Exception as e:
+            logger.error(f"Error: {e}")
+            if proc is not None:
+                proc.kill()
+                proc.wait()
+                logger.info("The process was terminated due to an error.")
+            raise  # Re-raise the exception to let the caller handle it
+
+        finally:
+            # 确保子进程已经结束
+            if proc is not None:
+                if proc.poll() is None:
+                    proc.kill()
+                    proc.wait()
+                    logger.info("The process was terminated due to timeout or completion.")
+
+    @staticmethod
+    def read_pdf_file(file_path) -> str:
+        try:
+            pdf_loader = PyMuPDFLoader(file_path)
+            pdf_documents = pdf_loader.load()
+            result_text = ''
+            for page in pdf_documents:
+                result_text += page.page_content
+            return result_text
+        except Exception as e:
+            raise e
+
+    def parse(self, file_path: str) -> MarkdownOutputVo:
+        try:
+            title = os.path.splitext(os.path.basename(file_path))[0]
+
+            if self.use_mineru:
+                output_dir = 'uploaded_files'
+                output_folder_name = os.path.basename(file_path).replace(".pdf", "")
+                # output_mineru = f'{output_dir}/{output_folder_name}/auto/{output_folder_name}.md'
+                # if os.path.exists(output_mineru):
+                #     pass
+                # else:
+                    # self.mineru_process(input_pdf_filename=file_path, output_dir=output_dir)
+                # mk_content = open(output_mineru, 'r', encoding='utf-8').read()
+
+                # todo: 是否有必要跟api的默认保存路径保持一致
+                output_mineru = f'{output_dir}/markdown/{output_folder_name}.md'
+
+                if os.path.exists(output_mineru):
+                    mk_content = open(output_mineru, 'r', encoding='utf-8').read()
+                else:
+                    mk_content = pdf_processor.process_pdf(file_path)
+            else:
+                content = self.read_pdf_file(file_path=file_path)
+                mk_content = content
+
+            lifecycle = self.generate_lifecycle(source_file=file_path, domain="Technology",
+                                                usage_purpose="Documentation", life_type="LLM_ORIGIN")
+            output_vo = MarkdownOutputVo(title, mk_content)
+            output_vo.add_lifecycle(lifecycle)
+            return output_vo.to_dict()
+        except Exception:
+            raise