pydatamax 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl

datamax/loader/core.py

@@ -1,7 +1,7 @@
 import os
 from typing import List
-from datamax.loader.MinioHandler import MinIOClient
-from datamax.loader.OssHandler import OssClient
+from datamax.loader.minio_handler import MinIOClient
+from datamax.loader.oss_handler import OssClient
 
 
 class DataLoader:

datamax/parser/doc_parser.py

@@ -1,14 +1,17 @@
+import logging
 import os
 import shutil
 import subprocess
 import tempfile
-import chardet
-import docx2markdown
 from pathlib import Path
 from typing import Union
-from docx import Document
-from datamax.parser.base import BaseLife
-from datamax.parser.base import MarkdownOutputVo
+
+import chardet
+
+from datamax.parser.base import BaseLife, MarkdownOutputVo
+
+# 配置日志
+logger = logging.getLogger(__name__)
 
 
 class DocParser(BaseLife):
@@ -16,62 +19,185 @@ class DocParser(BaseLife):
         super().__init__()
         self.file_path = file_path
         self.to_markdown = to_markdown
+        logger.info(f"🚀 DocParser初始化完成 - 文件路径: {file_path}, 转换为markdown: {to_markdown}")
+
+    def doc_to_txt(self, doc_path: str, dir_path: str) -> str:
+        """将.doc文件转换为.txt文件"""
+        logger.info(f"🔄 开始转换DOC文件为TXT - 源文件: {doc_path}, 输出目录: {dir_path}")
+
+        try:
+            cmd = f'soffice --headless --convert-to txt "{doc_path}" --outdir "{dir_path}"'
+            logger.debug(f"⚡ 执行转换命令: {cmd}")
+
+            process = subprocess.Popen(
+                cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            )
+            stdout, stderr = process.communicate()
+            exit_code = process.returncode
+
+            if exit_code == 0:
+                logger.info(f"✅ DOC到TXT转换成功 - 退出码: {exit_code}")
+                if stdout:
+                    logger.debug(f"📄 转换输出: {stdout.decode('utf-8', errors='replace')}")
+            else:
+                encoding = chardet.detect(stderr)["encoding"]
+                if encoding is None:
+                    encoding = "utf-8"
+                error_msg = stderr.decode(encoding, errors="replace")
+                logger.error(f"❌ DOC到TXT转换失败 - 退出码: {exit_code}, 错误信息: {error_msg}")
+                raise Exception(
+                    f"Error Output (detected encoding: {encoding}): {error_msg}"
+                )
+
+            fname = str(Path(doc_path).stem)
+            txt_path = os.path.join(dir_path, f"{fname}.txt")
+
+            if not os.path.exists(txt_path):
+                logger.error(f"❌ 转换后的TXT文件不存在: {txt_path}")
+                raise Exception(f"文件转换失败 {doc_path} ==> {txt_path}")
+            else:
+                logger.info(f"🎉 TXT文件转换成功，文件路径: {txt_path}")
+                return txt_path
+
+        except subprocess.SubprocessError as e:
+            logger.error(f"💥 subprocess执行失败: {str(e)}")
+            raise Exception(f"执行转换命令时发生错误: {str(e)}")
+        except Exception as e:
+            logger.error(f"💥 DOC到TXT转换过程中发生未知错误: {str(e)}")
+            raise
+
+    def read_txt_file(self, txt_path: str) -> str:
+        """读取txt文件内容"""
+        logger.info(f"📖 开始读取TXT文件: {txt_path}")
+
+        try:
+            # 检测文件编码
+            with open(txt_path, "rb") as f:
+                raw_data = f.read()
+                encoding = chardet.detect(raw_data)["encoding"]
+                if encoding is None:
+                    encoding = "utf-8"
+                logger.debug(f"🔍 检测到文件编码: {encoding}")
+
+            # 读取文件内容
+            with open(txt_path, "r", encoding=encoding, errors="replace") as f:
+                content = f.read()
+
+            logger.info(f"📄 TXT文件读取完成 - 内容长度: {len(content)} 字符")
+            logger.debug(f"👀 前100字符预览: {content[:100]}...")
+
+            return content
+
+        except FileNotFoundError as e:
+            logger.error(f"🚫 TXT文件未找到: {str(e)}")
+            raise Exception(f"文件未找到: {txt_path}")
+        except Exception as e:
+            logger.error(f"💥 读取TXT文件时发生错误: {str(e)}")
+            raise
+
+    def read_doc_file(self, doc_path: str) -> str:
+        """读取doc文件并转换为文本"""
+        logger.info(f"📖 开始读取DOC文件 - 文件: {doc_path}")
 
-    def doc_to_docx(self, doc_path: str, dir_path: str) -> str:
-        cmd = f'soffice --headless --convert-to docx "{doc_path}" --outdir "{dir_path}"'
-        process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout, stderr = process.communicate()
-        exit_code = process.returncode
-        if exit_code == 0:
-            pass
-        else:
-            encoding = chardet.detect(stderr)['encoding']
-            if encoding is None:
-                encoding = 'utf-8'
-            raise Exception(f"Error Output (detected encoding: {encoding}):", stderr.decode(encoding, errors='replace'))
-        fname = str(Path(doc_path).stem)
-        docx_path = os.path.join(os.path.dirname(doc_path), f'{fname}.docx')
-        if not os.path.exists(docx_path):
-            raise Exception(f"> !!! File conversion failed {doc_path} ==> {docx_path}")
-        else:
-            return docx_path
-
-    def read_docx_file(self, doc_path: str, to_mk: bool) -> str:
         try:
             with tempfile.TemporaryDirectory() as temp_path:
+                logger.debug(f"📁 创建临时目录: {temp_path}")
+
                 temp_dir = Path(temp_path)
-                media_dir = temp_dir / "media"
-                media_dir.mkdir()
+
                 file_path = temp_dir / "tmp.doc"
                 shutil.copy(doc_path, file_path)
-                docx_file_path = self.doc_to_docx(str(file_path), str(temp_path))
-                doc = Document(docx_file_path)
-                full_text = [para.text for para in doc.paragraphs]
-                if to_mk:
-                    output_md_dir = f'./output/{os.path.basename(docx_file_path).replace(".docx", ".md")}'
-                    output_dir = os.path.dirname(output_md_dir)
-                    if output_dir and not os.path.exists(output_dir):
-                        os.makedirs(output_dir)
-                    docx2markdown.docx_to_markdown(docx_file_path, output_md_dir)
-                    mk_content = open(output_md_dir, 'r', encoding='utf-8').read()
-                    return mk_content
-                else:
-                    return '\n'.join(full_text)
+                logger.debug(f"📋 复制文件到临时目录: {doc_path} -> {file_path}")
+
+                # 转换DOC为TXT
+                txt_file_path = self.doc_to_txt(str(file_path), str(temp_path))
+                logger.info(f"🎯 DOC转TXT完成: {txt_file_path}")
+
+                # 读取TXT文件内容
+                content = self.read_txt_file(txt_file_path)
+                logger.info(f"✨ TXT文件内容读取完成，内容长度: {len(content)} 字符")
+
+                return content
+
+        except FileNotFoundError as e:
+            logger.error(f"🚫 文件未找到: {str(e)}")
+            raise Exception(f"文件未找到: {doc_path}")
+        except PermissionError as e:
+            logger.error(f"🔒 文件权限错误: {str(e)}")
+            raise Exception(f"无权限访问文件: {doc_path}")
         except Exception as e:
-            raise e
+            logger.error(f"💥 读取DOC文件时发生错误: {str(e)}")
+            raise
 
     def parse(self, file_path: str):
+        """解析DOC文件"""
+        logger.info(f"🎬 开始解析DOC文件: {file_path}")
+
         try:
+            # 验证文件存在
+            if not os.path.exists(file_path):
+                logger.error(f"🚫 文件不存在: {file_path}")
+                raise FileNotFoundError(f"文件不存在: {file_path}")
+
+            # 验证文件大小
+            file_size = os.path.getsize(file_path)
+            logger.info(f"📏 文件大小: {file_size} 字节")
+
             title = self.get_file_extension(file_path)
+            logger.debug(f"🏷️ 提取文件标题: {title}")
+
+            # 使用soffice转换为txt后读取内容
+            logger.info("📝 使用soffice转换DOC为TXT并读取内容")
+            content = self.read_doc_file(doc_path=file_path)
+
+            # 根据to_markdown参数决定是否保持原格式还是处理为markdown格式
             if self.to_markdown:
-                mk_content = self.read_docx_file(doc_path=file_path, to_mk=True)
+                # 简单的文本到markdown转换（保持段落结构）
+                mk_content = self.format_as_markdown(content)
+                logger.info("🎨 内容已格式化为markdown格式")
             else:
-                content = self.read_docx_file(doc_path=file_path, to_mk=False)
                 mk_content = content
-            lifecycle = self.generate_lifecycle(source_file=file_path, domain="Technology",
-                                                usage_purpose="Documentation", life_type="LLM_ORIGIN")
+                logger.info("📝 保持原始文本格式")
+
+            logger.info(f"🎊 文件内容解析完成，最终内容长度: {len(mk_content)} 字符")
+
+            lifecycle = self.generate_lifecycle(
+                source_file=file_path,
+                domain="Technology",
+                usage_purpose="Documentation",
+                life_type="LLM_ORIGIN",
+            )
+            logger.debug("⚙️ 生成lifecycle信息完成")
+
             output_vo = MarkdownOutputVo(title, mk_content)
             output_vo.add_lifecycle(lifecycle)
-            return output_vo.to_dict()
+
+            result = output_vo.to_dict()
+            logger.info(f"🏆 DOC文件解析完成: {file_path}")
+            logger.debug(f"🔑 返回结果键: {list(result.keys())}")
+
+            return result
+
         except Exception as e:
-            raise e
+            logger.error(f"💀 解析DOC文件失败: {file_path}, 错误: {str(e)}")
+            raise
+
+    def format_as_markdown(self, content: str) -> str:
+        """将纯文本格式化为简单的markdown格式"""
+        if not content.strip():
+            return content
+
+        lines = content.split("\n")
+        formatted_lines = []
+
+        for line in lines:
+            line = line.strip()
+            if not line:
+                formatted_lines.append("")
+                continue
+
+            # 简单的markdown格式化规则
+            # 可以根据需要扩展更多规则
+            formatted_lines.append(line)
+
+        return "\n".join(formatted_lines)

datamax/parser/docx_parser.py

@@ -1,9 +1,17 @@
+import logging
 import os
-import docx2markdown
-from docx import Document
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
 from typing import Union
-from datamax.parser.base import BaseLife
-from datamax.parser.base import MarkdownOutputVo
+
+import chardet
+
+from datamax.parser.base import BaseLife, MarkdownOutputVo
+
+# 配置日志
+logger = logging.getLogger(__name__)
 
 
 class DocxParser(BaseLife):
@@ -11,33 +19,206 @@ class DocxParser(BaseLife):
         super().__init__()
         self.file_path = file_path
         self.to_markdown = to_markdown
+        logger.info(
+            f"🚀 DocxParser初始化完成 - 文件路径: {file_path}, 转换为markdown: {to_markdown}"
+        )
+
+    def docx_to_txt(self, docx_path: str, dir_path: str) -> str:
+        """将.docx文件转换为.txt文件"""
+        logger.info(f"🔄 开始转换DOCX文件为TXT - 源文件: {docx_path}, 输出目录: {dir_path}")
+
+        try:
+            cmd = f'soffice --headless --convert-to txt "{docx_path}" --outdir "{dir_path}"'
+            logger.debug(f"⚡ 执行转换命令: {cmd}")
+
+            process = subprocess.Popen(
+                cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            )
+            stdout, stderr = process.communicate()
+            exit_code = process.returncode
+
+            if exit_code == 0:
+                logger.info(f"✅ DOCX到TXT转换成功 - 退出码: {exit_code}")
+                if stdout:
+                    logger.debug(f"📄 转换输出: {stdout.decode('utf-8', errors='replace')}")
+            else:
+                encoding = chardet.detect(stderr)["encoding"]
+                if encoding is None:
+                    encoding = "utf-8"
+                error_msg = stderr.decode(encoding, errors="replace")
+                logger.error(f"❌ DOCX到TXT转换失败 - 退出码: {exit_code}, 错误信息: {error_msg}")
+                raise Exception(
+                    f"Error Output (detected encoding: {encoding}): {error_msg}"
+                )
+
+            fname = str(Path(docx_path).stem)
+            txt_path = os.path.join(dir_path, f"{fname}.txt")
+
+            if not os.path.exists(txt_path):
+                logger.error(f"❌ 转换后的TXT文件不存在: {txt_path}")
+                raise Exception(f"文件转换失败 {docx_path} ==> {txt_path}")
+            else:
+                logger.info(f"🎉 TXT文件转换成功，文件路径: {txt_path}")
+                return txt_path
+
+        except subprocess.SubprocessError as e:
+            logger.error(f"💥 subprocess执行失败: {str(e)}")
+            raise Exception(f"执行转换命令时发生错误: {str(e)}")
+        except Exception as e:
+            logger.error(f"💥 DOCX到TXT转换过程中发生未知错误: {str(e)}")
+            raise
+
+    def read_txt_file(self, txt_path: str) -> str:
+        """读取txt文件内容"""
+        logger.info(f"📖 开始读取TXT文件: {txt_path}")
+
+        try:
+            # 检测文件编码
+            with open(txt_path, "rb") as f:
+                raw_data = f.read()
+                encoding = chardet.detect(raw_data)["encoding"]
+                if encoding is None:
+                    encoding = "utf-8"
+                logger.debug(f"🔍 检测到文件编码: {encoding}")
+
+            # 读取文件内容
+            with open(txt_path, "r", encoding=encoding, errors="replace") as f:
+                content = f.read()
+
+            logger.info(f"📄 TXT文件读取完成 - 内容长度: {len(content)} 字符")
+            logger.debug(f"👀 前100字符预览: {content[:100]}...")
+
+            return content
+
+        except FileNotFoundError as e:
+            logger.error(f"🚫 TXT文件未找到: {str(e)}")
+            raise Exception(f"文件未找到: {txt_path}")
+        except Exception as e:
+            logger.error(f"💥 读取TXT文件时发生错误: {str(e)}")
+            raise
+
+    def read_docx_file(self, docx_path: str) -> str:
+        """读取docx文件并转换为文本"""
+        logger.info(f"📖 开始读取DOCX文件 - 文件: {docx_path}")
 
-    @staticmethod
-    def read_docx_file(file_path: str) -> str:
         try:
-            doc = Document(file_path)
-            full_text = [para.text for para in doc.paragraphs]
-            return '\n'.join(full_text)
+            with tempfile.TemporaryDirectory() as temp_path:
+                logger.debug(f"📁 创建临时目录: {temp_path}")
+
+                temp_dir = Path(temp_path)
+
+                file_path = temp_dir / "tmp.docx"
+                shutil.copy(docx_path, file_path)
+                logger.debug(f"📋 复制文件到临时目录: {docx_path} -> {file_path}")
+
+                # 转换DOCX为TXT
+                txt_file_path = self.docx_to_txt(str(file_path), str(temp_path))
+                logger.info(f"🎯 DOCX转TXT完成: {txt_file_path}")
+
+                # 读取TXT文件内容
+                content = self.read_txt_file(txt_file_path)
+                logger.info(f"✨ TXT文件内容读取完成，内容长度: {len(content)} 字符")
+
+                return content
+
+        except FileNotFoundError as e:
+            logger.error(f"🚫 文件未找到: {str(e)}")
+            raise Exception(f"文件未找到: {docx_path}")
+        except PermissionError as e:
+            logger.error(f"🔒 文件权限错误: {str(e)}")
+            raise Exception(f"无权限访问文件: {docx_path}")
         except Exception as e:
-            raise e
+            logger.error(f"💥 读取DOCX文件时发生错误: {str(e)}")
+            raise
 
     def parse(self, file_path: str):
+        """解析DOCX文件"""
+        logger.info(f"🎬 开始解析DOCX文件: {file_path}")
+
         try:
+            # 验证文件存在
+            if not os.path.exists(file_path):
+                logger.error(f"🚫 文件不存在: {file_path}")
+                raise FileNotFoundError(f"文件不存在: {file_path}")
+
+            # 验证文件扩展名
+            if not file_path.lower().endswith(".docx"):
+                logger.warning(f"⚠️ 文件扩展名不是.docx: {file_path}")
+
+            # 验证文件大小
+            file_size = os.path.getsize(file_path)
+            logger.info(f"📏 文件大小: {file_size} 字节")
+
+            if file_size == 0:
+                logger.warning(f"⚠️ 文件大小为0字节: {file_path}")
+
             title = self.get_file_extension(file_path)
+            logger.debug(f"🏷️ 提取文件标题: {title}")
+
+            # 使用soffice转换为txt后读取内容
+            logger.info("📝 使用soffice转换DOCX为TXT并读取内容")
+            content = self.read_docx_file(docx_path=file_path)
+
+            # 根据to_markdown参数决定是否保持原格式还是处理为markdown格式
             if self.to_markdown:
-                output_md_dir = f'./output/{os.path.basename(file_path).replace(".docx", ".md")}'
-                output_dir = os.path.dirname(output_md_dir)
-                if output_dir and not os.path.exists(output_dir):
-                    os.makedirs(output_dir)
-                docx2markdown.docx_to_markdown(file_path, output_md_dir)
-                mk_content = open(output_md_dir, 'r', encoding='utf-8').read()
+                # 简单的文本到markdown转换（保持段落结构）
+                mk_content = self.format_as_markdown(content)
+                logger.info("🎨 内容已格式化为markdown格式")
             else:
-                content = self.read_docx_file(file_path=file_path)
                 mk_content = content
-            lifecycle = self.generate_lifecycle(source_file=file_path, domain="Technology",
-                                                usage_purpose="Documentation", life_type="LLM_ORIGIN")
+                logger.info("📝 保持原始文本格式")
+
+            logger.info(f"🎊 文件内容解析完成，最终内容长度: {len(mk_content)} 字符")
+
+            # 检查内容是否为空
+            if not mk_content.strip():
+                logger.warning(f"⚠️ 解析出的内容为空: {file_path}")
+
+            lifecycle = self.generate_lifecycle(
+                source_file=file_path,
+                domain="Technology",
+                usage_purpose="Documentation",
+                life_type="LLM_ORIGIN",
+            )
+            logger.debug("⚙️ 生成lifecycle信息完成")
+
             output_vo = MarkdownOutputVo(title, mk_content)
             output_vo.add_lifecycle(lifecycle)
-            return output_vo.to_dict()
+
+            result = output_vo.to_dict()
+            logger.info(f"🏆 DOCX文件解析完成: {file_path}")
+            logger.debug(f"🔑 返回结果键: {list(result.keys())}")
+
+            return result
+
+        except FileNotFoundError as e:
+            logger.error(f"🚫 文件不存在错误: {str(e)}")
+            raise
+        except PermissionError as e:
+            logger.error(f"🔒 文件权限错误: {str(e)}")
+            raise Exception(f"无权限访问文件: {file_path}")
         except Exception as e:
-            raise e
+            logger.error(
+                f"💀 解析DOCX文件失败: {file_path}, 错误类型: {type(e).__name__}, 错误信息: {str(e)}"
+            )
+            raise
+
+    def format_as_markdown(self, content: str) -> str:
+        """将纯文本格式化为简单的markdown格式"""
+        if not content.strip():
+            return content
+
+        lines = content.split("\n")
+        formatted_lines = []
+
+        for line in lines:
+            line = line.strip()
+            if not line:
+                formatted_lines.append("")
+                continue
+
+            # 简单的markdown格式化规则
+            # 可以根据需要扩展更多规则
+            formatted_lines.append(line)
+
+        return "\n".join(formatted_lines)

datamax/parser/xlsx_parser.py

@@ -1,71 +1,215 @@
+import logging
 import multiprocessing
+import os
 import time
-from multiprocessing import Queue
-from datamax.parser.base import MarkdownOutputVo
-from datamax.parser.base import BaseLife
-from openpyxl import load_workbook
 import warnings
-from markitdown import MarkItDown
+from multiprocessing import Queue
+
+import pandas as pd
+
+from datamax.parser.base import BaseLife, MarkdownOutputVo
 
 warnings.filterwarnings("ignore")
 
-class XlsxParser(BaseLife):
-    # single ton
+# 配置日志
+logger = logging.getLogger(__name__)
 
-    _markitdown_instance = None
 
-    @classmethod
-    def get_markitdown(cls):
-        if cls._markitdown_instance is None:
-            cls._markitdown_instance = MarkItDown()
-        return cls._markitdown_instance
+class XlsxParser(BaseLife):
+    """XLSX解析器 - 使用pandas读取并转换为markdown，支持多进程处理"""
 
     def __init__(self, file_path, timeout):
         super().__init__()
         self.file_path = file_path
         self.timeout = timeout
-        self.markitdown = self.get_markitdown()
+        logger.info(f"🚀 XlsxParser初始化完成 - 文件路径: {file_path}, 超时: {timeout}s")
+
+    def _parse_with_pandas(self, file_path: str) -> str:
+        """使用pandas读取Excel并转换为markdown"""
+        logger.info(f"🐼 开始使用pandas读取Excel文件: {file_path}")
+
+        try:
+            # 验证文件存在
+            if not os.path.exists(file_path):
+                logger.error(f"🚫 Excel文件不存在: {file_path}")
+                raise FileNotFoundError(f"文件不存在: {file_path}")
+
+            # 验证文件大小
+            file_size = os.path.getsize(file_path)
+            logger.info(f"📏 文件大小: {file_size} 字节")
+
+            if file_size == 0:
+                logger.warning(f"⚠️ 文件大小为0字节: {file_path}")
+                return "*文件为空*"
+
+            # 使用pandas读取Excel文件
+            logger.debug("📊 正在读取Excel数据...")
+            df = pd.read_excel(file_path, sheet_name=None)  # 读取所有工作表
+
+            markdown_content = ""
+
+            if isinstance(df, dict):
+                # 多个工作表
+                logger.info(f"📑 检测到多个工作表，共 {len(df)} 个")
+                for sheet_name, sheet_df in df.items():
+                    logger.debug(f"📋 处理工作表: {sheet_name}, 形状: {sheet_df.shape}")
+                    markdown_content += f"## 工作表: {sheet_name}\n\n"
+
+                    if not sheet_df.empty:
+                        # 清理数据：移除完全为空的行和列
+                        sheet_df = sheet_df.dropna(how="all").dropna(axis=1, how="all")
+
+                        if not sheet_df.empty:
+                            sheet_markdown = sheet_df.to_markdown(index=False)
+                            markdown_content += sheet_markdown + "\n\n"
+                            logger.debug(
+                                f"✅ 工作表 {sheet_name} 转换完成，有效数据形状: {sheet_df.shape}"
+                            )
+                        else:
+                            markdown_content += "*该工作表无有效数据*\n\n"
+                            logger.warning(f"⚠️ 工作表 {sheet_name} 清理后无有效数据")
+                    else:
+                        markdown_content += "*该工作表为空*\n\n"
+                        logger.warning(f"⚠️ 工作表 {sheet_name} 为空")
+            else:
+                # 单个工作表
+                logger.info(f"📄 单个工作表，形状: {df.shape}")
+                if not df.empty:
+                    # 清理数据：移除完全为空的行和列
+                    df = df.dropna(how="all").dropna(axis=1, how="all")
+
+                    if not df.empty:
+                        markdown_content = df.to_markdown(index=False)
+                        logger.info(f"✅ 工作表转换完成，有效数据形状: {df.shape}")
+                    else:
+                        markdown_content = "*工作表无有效数据*"
+                        logger.warning("⚠️ 工作表清理后无有效数据")
+                else:
+                    markdown_content = "*工作表为空*"
+                    logger.warning("⚠️ 工作表为空")
+
+            logger.info(f"🎊 pandas转换完成，markdown内容长度: {len(markdown_content)} 字符")
+            logger.debug(f"👀 前200字符预览: {markdown_content[:200]}...")
+
+            return markdown_content
+
+        except FileNotFoundError as e:
+            logger.error(f"🚫 文件未找到: {str(e)}")
+            raise
+        except PermissionError as e:
+            logger.error(f"🔒 文件权限错误: {str(e)}")
+            raise Exception(f"无权限访问文件: {file_path}")
+        except pd.errors.EmptyDataError as e:
+            logger.error(f"📭 Excel文件为空: {str(e)}")
+            raise Exception(f"Excel文件为空或无法读取: {file_path}")
+        except Exception as e:
+            logger.error(f"💥 pandas读取Excel失败: {str(e)}")
+            raise
 
     def _parse(self, file_path: str, result_queue: Queue) -> dict:
+        """解析Excel文件的核心方法"""
+        logger.info(f"🎬 开始解析Excel文件: {file_path}")
+
         try:
-            wb = load_workbook(
-                filename=file_path,
-                data_only=True,
-                read_only=True
+            # 使用pandas解析Excel
+            logger.info("🐼 使用pandas模式解析Excel")
+            mk_content = self._parse_with_pandas(file_path)
+
+            # 检查内容是否为空
+            if not mk_content.strip():
+                logger.warning(f"⚠️ 解析出的内容为空: {file_path}")
+                mk_content = "*无法解析文件内容*"
+
+            logger.info(f"🎊 文件内容解析完成，最终内容长度: {len(mk_content)} 字符")
+
+            # 生成lifecycle信息
+            lifecycle = self.generate_lifecycle(
+                source_file=file_path,
+                domain="Technology",
+                usage_purpose="Documentation",
+                life_type="LLM_ORIGIN",
             )
-            wb.close()
+            logger.debug("⚙️ 生成lifecycle信息完成")
+
+            # 创建输出对象
+            title = self.get_file_extension(file_path)
+            output_vo = MarkdownOutputVo(title, mk_content)
+            output_vo.add_lifecycle(lifecycle)
+
+            result = output_vo.to_dict()
+            result_queue.put(result)
+            logger.info(f"🏆 Excel文件解析完成: {file_path}")
+            logger.debug(f"🔑 返回结果键: {list(result.keys())}")
+
+            time.sleep(0.5)  # 给队列一点时间
+            return result
+
         except Exception as e:
-            raise e
-
-        mk_content = self.markitdown.convert(file_path).text_content
-        lifecycle = self.generate_lifecycle(
-            source_file=file_path,
-            domain="Technology",
-            usage_purpose="Documentation",
-            life_type="LLM_ORIGIN"
-        )
-        output_vo = MarkdownOutputVo(self.get_file_extension(file_path), mk_content)
-        output_vo.add_lifecycle(lifecycle)
-        result_queue.put(output_vo.to_dict())
-        time.sleep(0.5)
-        return output_vo.to_dict()
+            logger.error(f"💀 解析Excel文件失败: {file_path}, 错误: {str(e)}")
+            # 将错误也放入队列
+            error_result = {"error": str(e), "file_path": file_path}
+            result_queue.put(error_result)
+            raise
 
     def parse(self, file_path: str) -> dict:
-        import time
-        result_queue = Queue()
-        process = multiprocessing.Process(target=self._parse, args=(file_path, result_queue))
-        process.start()
-        start_time = time.time()
-
-        # ttl
-        while time.time() - start_time < self.timeout:
-            print(f"plz waiting...: {int(time.time() - start_time)}")
-            if not process.is_alive():
-                break
-            if not result_queue.empty():
-                return result_queue.get()
-            time.sleep(1)
-        else:
-            # killed
-            process.terminate()
-            process.join()
+        """解析Excel文件 - 支持多进程和超时控制"""
+        logger.info(f"🚀 启动Excel解析进程 - 文件: {file_path}, 超时: {self.timeout}s")
+
+        try:
+            # 验证文件存在
+            if not os.path.exists(file_path):
+                logger.error(f"🚫 文件不存在: {file_path}")
+                raise FileNotFoundError(f"文件不存在: {file_path}")
+
+            # 验证文件扩展名
+            if not file_path.lower().endswith((".xlsx", ".xls")):
+                logger.warning(f"⚠️ 文件扩展名不是Excel格式: {file_path}")
+
+            result_queue = Queue()
+            process = multiprocessing.Process(
+                target=self._parse, args=(file_path, result_queue)
+            )
+            process.start()
+            logger.debug(f"⚡ 启动子进程，PID: {process.pid}")
+
+            start_time = time.time()
+
+            # 等待解析完成或超时
+            while time.time() - start_time < self.timeout:
+                elapsed_time = int(time.time() - start_time)
+                logger.debug(f"⏱️ 等待解析完成... {elapsed_time}s")
+
+                if not process.is_alive():
+                    logger.debug("✅ 子进程已完成")
+                    break
+
+                if not result_queue.empty():
+                    result = result_queue.get()
+                    process.join()  # 等待进程正常结束
+
+                    # 检查是否是错误结果
+                    if "error" in result:
+                        logger.error(f"💥 子进程返回错误: {result['error']}")
+                        raise Exception(result["error"])
+
+                    logger.info(f"🎉 Excel解析成功完成，耗时: {elapsed_time}s")
+                    return result
+
+                time.sleep(1)
+            else:
+                # 超时处理
+                logger.error(f"⏰ 解析超时 ({self.timeout}s)，终止进程")
+                process.terminate()
+                process.join(timeout=5)  # 给进程5秒时间优雅退出
+
+                if process.is_alive():
+                    logger.error("💀 强制杀死进程")
+                    process.kill()
+
+                raise TimeoutError(f"Excel解析超时: {file_path}")
+
+        except Exception as e:
+            logger.error(
+                f"💀 Excel解析失败: {file_path}, 错误类型: {type(e).__name__}, 错误信息: {str(e)}"
+            )
+            raise

pydatamax-0.1.14.dist-info/METADATA

@@ -0,0 +1,228 @@
+Metadata-Version: 2.4
+Name: pydatamax
+Version: 0.1.14
+Summary: A library for parsing and converting various file formats.
+Home-page: https://github.com/Hi-Dolphin/datamax
+Author: ccy
+Author-email: cy.kron@foxmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: oss2<3.0.0,>=2.19.1
+Requires-Dist: aliyun-python-sdk-core<3.0.0,>=2.16.0
+Requires-Dist: aliyun-python-sdk-kms<3.0.0,>=2.16.5
+Requires-Dist: crcmod<2.0.0,>=1.7
+Requires-Dist: langdetect<2.0.0,>=1.0.9
+Requires-Dist: loguru<1.0.0,>=0.7.3
+Requires-Dist: python-docx<2.0.0,>=1.1.2
+Requires-Dist: python-dotenv<2.0.0,>=1.1.0
+Requires-Dist: pymupdf<2.0.0,>=1.26.0
+Requires-Dist: pypdf<6.0.0,>=5.5.0
+Requires-Dist: openpyxl<4.0.0,>=3.1.5
+Requires-Dist: pandas<3.0.0,>=2.2.3
+Requires-Dist: numpy<3.0.0,>=2.2.6
+Requires-Dist: requests<3.0.0,>=2.32.3
+Requires-Dist: tqdm<5.0.0,>=4.67.1
+Requires-Dist: pydantic<3.0.0,>=2.11.5
+Requires-Dist: pydantic-settings<3.0.0,>=2.9.1
+Requires-Dist: python-magic<1.0.0,>=0.4.27
+Requires-Dist: PyYAML<7.0.0,>=6.0.2
+Requires-Dist: Pillow<12.0.0,>=11.2.1
+Requires-Dist: packaging<25.0,>=24.2
+Requires-Dist: beautifulsoup4<5.0.0,>=4.13.4
+Requires-Dist: minio<8.0.0,>=7.2.15
+Requires-Dist: openai<2.0.0,>=1.82.0
+Requires-Dist: jionlp<2.0.0,>=1.5.23
+Requires-Dist: chardet<6.0.0,>=5.2.0
+Requires-Dist: python-pptx<2.0.0,>=1.0.2
+Requires-Dist: tiktoken<1.0.0,>=0.9.0
+Requires-Dist: markitdown<1.0.0,>=0.1.1
+Requires-Dist: xlrd<3.0.0,>=2.0.1
+Requires-Dist: tabulate<1.0.0,>=0.9.0
+Requires-Dist: unstructured<1.0.0,>=0.17.2
+Requires-Dist: markdown<4.0.0,>=3.8
+Requires-Dist: langchain<1.0.0,>=0.3.0
+Requires-Dist: langchain-community<1.0.0,>=0.3.0
+Requires-Dist: ebooklib==0.19
+Requires-Dist: setuptools
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# DataMax
+
+<div align="center">
+
+[中文](README_zh.md) | **English**
+
+[![PyPI version](https://badge.fury.io/py/pydatamax.svg)](https://badge.fury.io/py/pydatamax) [![Python](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+</div>
+
+A powerful multi-format file parsing, data cleaning, and AI annotation toolkit.
+
+## ✨ Core Features
+
+- 🔄 **Multi-format Support**: PDF, DOCX/DOC, PPT/PPTX, XLS/XLSX, HTML, EPUB, TXT, images, and more
+- 🧹 **Intelligent Cleaning**: Three-layer cleaning process with anomaly detection, privacy protection, and text filtering
+- 🤖 **AI Annotation**: LLM-based automatic data annotation and pre-labeling
+- ⚡ **Batch Processing**: Efficient multi-file parallel processing
+- 🎯 **Easy Integration**: Clean API design, ready to use out of the box
+
+## 🚀 Quick Start
+
+### Installation
+
+```bash
+pip install pydatamax
+```
+
+### Basic Usage
+
+```python
+from datamax import DataMax
+
+# Parse a single file
+dm = DataMax(file_path="document.pdf")
+data = dm.get_data()
+
+# Batch processing
+dm = DataMax(file_path=["file1.docx", "file2.pdf"])
+data = dm.get_data()
+
+# Data cleaning
+cleaned_data = dm.clean_data(method_list=["abnormal", "private", "filter"])
+
+# AI annotation
+qa_data = dm.get_pre_label(
+    api_key="your-api-key",
+    base_url="https://api.openai.com/v1",
+    model_name="gpt-3.5-turbo"
+)
+```
+
+## 📖 Detailed Documentation
+
+### File Parsing
+
+#### Supported Formats
+
+| Format | Extensions | Special Features |
+|--------|------------|------------------|
+| Documents | `.pdf`, `.docx`, `.doc` | OCR support, Markdown conversion |
+| Spreadsheets | `.xlsx`, `.xls` | Structured data extraction |
+| Presentations | `.pptx`, `.ppt` | Slide content extraction |
+| Web | `.html`, `.epub` | Tag parsing |
+| Images | `.jpg`, `.png`, `.jpeg` | OCR text recognition |
+| Text | `.txt` | Automatic encoding detection |
+
+#### Advanced Features
+
+```python
+# Advanced PDF parsing (requires MinerU)
+dm = DataMax(file_path="complex.pdf", use_mineru=True)
+
+# Word to Markdown conversion
+dm = DataMax(file_path="document.docx", to_markdown=True)
+
+# Image OCR
+dm = DataMax(file_path="image.jpg", use_ocr=True)
+```
+
+### Data Cleaning
+
+```python
+# Three cleaning modes
+dm.clean_data(method_list=[
+    "abnormal",  # Anomaly data processing
+    "private",   # Privacy information masking
+    "filter"     # Text filtering and normalization
+])
+```
+
+### AI Annotation
+
+```python
+# Custom annotation tasks
+qa_data = dm.get_pre_label(
+    api_key="sk-xxx",
+    base_url="https://api.provider.com/v1",
+    model_name="model-name",
+    chunk_size=500,        # Text chunk size
+    chunk_overlap=100,     # Overlap length
+    question_number=5,     # Questions per chunk
+    max_workers=5          # Concurrency
+)
+```
+
+## ⚙️ Environment Setup
+
+### Optional Dependencies
+
+#### LibreOffice (DOC file support)
+
+**Ubuntu/Debian:**
+```bash
+sudo apt-get install libreoffice
+```
+
+**Windows:**
+1. Download and install [LibreOffice](https://www.libreoffice.org/download/)
+2. Add to environment variables: `C:\Program Files\LibreOffice\program`
+
+#### MinerU (Advanced PDF parsing)
+
+```bash
+# Create virtual environment
+conda create -n mineru python=3.10
+conda activate mineru
+
+# Install MinerU
+pip install -U "magic-pdf[full]" --extra-index-url https://wheels.myhloli.com
+```
+
+For detailed configuration, please refer to [MinerU Documentation](https://github.com/opendatalab/MinerU)
+
+## 🛠️ Development
+
+### Local Installation
+
+```bash
+git clone https://github.com/Hi-Dolphin/datamax.git
+cd datamax
+pip install -r requirements.txt
+python setup.py install
+```
+
+## 📋 System Requirements
+
+- Python >= 3.10
+- Supports Windows, macOS, Linux
+
+## 🤝 Contributing
+
+Issues and Pull Requests are welcome!
+
+## 📄 License
+
+This project is licensed under the [MIT License](LICENSE).
+
+## 📞 Contact Us
+
+- 📧 Email: cy.kron@foxmail.com
+- 🐛 Issues: [GitHub Issues](https://github.com/Hi-Dolphin/datamax/issues)
+- 📚 Documentation: [Project Homepage](https://github.com/Hi-Dolphin/datamax)
+
+---
+
+⭐ If this project helps you, please give us a star! 

{pydatamax-0.1.12.dist-info → pydatamax-0.1.14.dist-info}/RECORD

@@ -1,14 +1,14 @@
 datamax/__init__.py,sha256=Kbs8ITE6suPy0VL8WzKH8A_iAGqukC0jIHcFGLgoBw8,28
-datamax/loader/MinioHandler.py,sha256=0XLvtKayhI2cjPU7S7m91tSzPcaOC924ByJNbabY6So,6386
-datamax/loader/OssHandler.py,sha256=YFbUfH2vkKRVg8OhkYpOrlYHrIX4zofbHFVe8nPpsQ4,7457
 datamax/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamax/loader/core.py,sha256=tSIkOw5D3EVFYme1b7joFt0e_LxJdf-mdUzxpyVt0VI,5098
+datamax/loader/core.py,sha256=C5VhDXkv0HTRcF6WWtztatUZHURZBD-KoZpKyqXfD7U,5100
+datamax/loader/minio_handler.py,sha256=0XLvtKayhI2cjPU7S7m91tSzPcaOC924ByJNbabY6So,6386
+datamax/loader/oss_handler.py,sha256=YFbUfH2vkKRVg8OhkYpOrlYHrIX4zofbHFVe8nPpsQ4,7457
 datamax/parser/__init__.py,sha256=Jilq2PLBNonmoXKATzsIHWWvFuBdlcV2dbSP1cOZ6zg,111
 datamax/parser/base.py,sha256=riGcMn4m295_qf9O0-NbHU2BcHGBXvoF4T3fWj9vgUQ,2514
 datamax/parser/core.py,sha256=9rzIjsVTRacPTUTAVa5gm5fx0h95LxYnw0lEGqjIIB4,11437
 datamax/parser/csv_parser.py,sha256=IcyVq8mGE6auIcUInXGYWDnh0H0XJ_3SyQrLVRrS7i0,190
-datamax/parser/doc_parser.py,sha256=VwTOdq5pGPbOI-98SoTwwTcXIjD1BrZDfFGEhTi3T44,3348
-datamax/parser/docx_parser.py,sha256=OhqcMeZ8JkwDJtvrMirM15j-EDnGNUj6U1-nX3gisKA,1727
+datamax/parser/doc_parser.py,sha256=wVNYn7dkPI12pW_YImDgoceLFWS3RvFpzbFQwVlrnNo,7936
+datamax/parser/docx_parser.py,sha256=o9_1VBDc8nBCmsEMv0sKsVcPMxiAxY5pl0mUvOcoOJc,8796
 datamax/parser/epub_parser.py,sha256=ljCGxLBPwE5gXVKARJec93VpP4dE9R2GspzuSZBkqPQ,1557
 datamax/parser/html_parser.py,sha256=xQaaK8674QbQwE-Up9X0DJIH0Gg0mR2KoI7fJ6iw2m0,1393
 datamax/parser/image_parser.py,sha256=qGCndc_21PwsfuxFG03wHSsV0uc-XMBaW3VDbsJQd90,1233
@@ -19,7 +19,7 @@ datamax/parser/ppt_parser.py,sha256=Niu3Ina6I6m6lAMS1Z-A7rUbR_iFGmNTaASBoNH_vZ0,
 datamax/parser/pptx_parser.py,sha256=sFWyOa3QNIs4BgtpmSzFQgsgPmunfGqCqi6fulbLFW0,1811
 datamax/parser/txt_parser.py,sha256=4DIP1LVOw21NDdtqG2RTD_hMcHufkvC8kr048AkuLFs,1682
 datamax/parser/xls_parser.py,sha256=pRlqgg96f76H8UqXQfheQT9O0ThdP7958hKUCEyQfPM,954
-datamax/parser/xlsx_parser.py,sha256=9ZqwCSF01thjEb_RleWGCiNOSuA8KZ3QFqzUKldb3wE,2183
+datamax/parser/xlsx_parser.py,sha256=Vw6XfoQyu6aQUSIueR-krByMA_WOb5fasf4VmKxjVio,8905
 datamax/utils/__init__.py,sha256=d69SJvqOXzItyg9rEcLc4z67Lw9vACispOe3x7NvZLA,1051
 datamax/utils/constants.py,sha256=A0S56mkIfeT6oQmOd-VGTChzLOSBUqsG4skMmLt6uNk,4507
 datamax/utils/data_cleaner.py,sha256=zlk2dXmhU-_9KVfqmqMGr967v-nc7Iv8ZKRdMkIJsGM,7784
@@ -30,10 +30,10 @@ datamax/utils/paddleocr_pdf_operator.py,sha256=Tnb-5SzUd6OXM-XeaL8vdPnsOhgG_GKz-
 datamax/utils/ppt_extract.py,sha256=nd6KSqEzxANrPhNPUZY4ogAyxHzKCbdsI5ZfDQCz0Cw,6164
 datamax/utils/qa_generator.py,sha256=d75an9JEyT6sxlSjdmWYveQshfyTb0v4aGSuTpTJa0A,12561
 datamax/utils/tokenizer.py,sha256=Y8XB06XQVsNuG8IPl_4iBZj2yu1xzXldVbmZtXFMQM4,859
-pydatamax-0.1.12.dist-info/licenses/LICENSE,sha256=LvCq2Pc7MejIvfNeRl_kAM5l_KXWlQFiC-Sjp7kqFf8,1067
+pydatamax-0.1.14.dist-info/licenses/LICENSE,sha256=LvCq2Pc7MejIvfNeRl_kAM5l_KXWlQFiC-Sjp7kqFf8,1067
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_basic.py,sha256=4AByx25-MIt6_zmzxpFRoSCBqLtIjyfTwFLb1UCJz6k,303
-pydatamax-0.1.12.dist-info/METADATA,sha256=6D9bCVBg8aNF1kg4gnXrPe7ybvMEWFp7MKxl2_Ivb4s,9774
-pydatamax-0.1.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pydatamax-0.1.12.dist-info/top_level.txt,sha256=DvdVIUrxJDgRUYiGA5vznYZIP-K8ZnDkTZfrqYLNZMQ,14
-pydatamax-0.1.12.dist-info/RECORD,,
+pydatamax-0.1.14.dist-info/METADATA,sha256=P-wz8Log3gcUMftKTd2qrcmNuzpp-HOn8gVVe8cTceM,6314
+pydatamax-0.1.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pydatamax-0.1.14.dist-info/top_level.txt,sha256=DvdVIUrxJDgRUYiGA5vznYZIP-K8ZnDkTZfrqYLNZMQ,14
+pydatamax-0.1.14.dist-info/RECORD,,

pydatamax-0.1.12.dist-info/METADATA

@@ -1,281 +0,0 @@
-Metadata-Version: 2.4
-Name: pydatamax
-Version: 0.1.12
-Summary: A library for parsing and converting various file formats.
-Home-page: https://github.com/Hi-Dolphin/datamax
-Author: ccy
-Author-email: cy.kron@foxmail.com
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: oss2<3.0.0,>=2.19.1
-Requires-Dist: aliyun-python-sdk-core<3.0.0,>=2.16.0
-Requires-Dist: aliyun-python-sdk-kms<3.0.0,>=2.16.5
-Requires-Dist: crcmod<2.0.0,>=1.7
-Requires-Dist: langdetect<2.0.0,>=1.0.9
-Requires-Dist: loguru<1.0.0,>=0.7.3
-Requires-Dist: python-docx<2.0.0,>=1.1.2
-Requires-Dist: python-dotenv<2.0.0,>=1.1.0
-Requires-Dist: pymupdf<2.0.0,>=1.26.0
-Requires-Dist: pypdf<6.0.0,>=5.5.0
-Requires-Dist: openpyxl<4.0.0,>=3.1.5
-Requires-Dist: pandas<3.0.0,>=2.2.3
-Requires-Dist: numpy<3.0.0,>=2.2.6
-Requires-Dist: requests<3.0.0,>=2.32.3
-Requires-Dist: tqdm<5.0.0,>=4.67.1
-Requires-Dist: pydantic<3.0.0,>=2.11.5
-Requires-Dist: pydantic-settings<3.0.0,>=2.9.1
-Requires-Dist: python-magic<1.0.0,>=0.4.27
-Requires-Dist: PyYAML<7.0.0,>=6.0.2
-Requires-Dist: Pillow<12.0.0,>=11.2.1
-Requires-Dist: packaging<25.0,>=24.2
-Requires-Dist: beautifulsoup4<5.0.0,>=4.13.4
-Requires-Dist: minio<8.0.0,>=7.2.15
-Requires-Dist: openai<2.0.0,>=1.82.0
-Requires-Dist: jionlp<2.0.0,>=1.5.23
-Requires-Dist: chardet<6.0.0,>=5.2.0
-Requires-Dist: python-pptx<2.0.0,>=1.0.2
-Requires-Dist: docx2markdown<1.0.0,>=0.1.1
-Requires-Dist: tiktoken<1.0.0,>=0.9.0
-Requires-Dist: markitdown<1.0.0,>=0.1.1
-Requires-Dist: xlrd<3.0.0,>=2.0.1
-Requires-Dist: tabulate<1.0.0,>=0.9.0
-Requires-Dist: unstructured<1.0.0,>=0.17.2
-Requires-Dist: markdown<4.0.0,>=3.8
-Requires-Dist: langchain<1.0.0,>=0.3.0
-Requires-Dist: langchain-community<1.0.0,>=0.3.0
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license-file
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary
-
-# DataMax
-
-## Overview
-DataMax is designed as a comprehensive solution for processing diverse file formats, performing data cleaning, and facilitating data annotation.
-
-## Key Features
-
-### File Processing Capabilities
-Currently supports reading, conversion, and extraction from:
-- PDF, HTML  
-- DOCX/DOC, PPT/PPTX  
-- EPUB  
-- Images
-- XLS/XLSX spreadsheets  
-- Plain text (TXT)  
-
-### Data Cleaning Pipeline
-Three-tiered cleaning process:
-1. Anomaly detection and handling  
-2. Privacy protection processing  
-3. Text filtering and normalization  
-
-### AI-Powered Data Annotation
-Implements an LLM+Prompt to:
-- Continuously generate pre-labeled datasets  
-- Provide optimized training data for model fine-tuning  
-
-
-## Installation Guide (Key Dependencies)
-Dependencies include libreoffice, datamax, and MinerU.
-
-### 1. Installing libreoffice Dependency
-**Note:** Without datamax, .doc files will not be supported.
-
-#### Linux (Debian/Ubuntu)
-```bash
-sudo apt-get update
-sudo apt-get install libreoffice
-```
-### Windows
-```text
-Install LibreOffice from: [Download LibreOffice](https://www.libreoffice.org/download/download-libreoffice/?spm=5176.28103460.0.0.5b295d275bpHzh)  
-Add to environment variable: `$env:PATH += ";C:\Program Files\LibreOffice\program"`
-```
-### Checking LibreOffice Installation
-```bash
-soffice --version
-```
-
-## 2. Installing MinerU Dependency
-Note: Without MinerU, advanced OCR parsing for PDFs will not be supported.
-### Create a Virtual Environment and Install Basic Dependencies
-```bash
-conda create -n mineru python=3.10
-conda activate mineru
-pip install -U "magic-pdf[full]" --extra-index-url https://wheels.myhloli.com -i https://mirrors.aliyun.com/pypi/simple
-```
-### Installing Model Weight Files
-https://github.com/opendatalab/MinerU/blob/master/docs/how_to_download_models_zh_cn.md
-```bash
-pip install modelscope
-wget https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/scripts/download_models.py -O download_models.py
-python download_models.py
-```
-
-### Modify the Configuration File magic-pdf.json (Located in the User Directory, Template Preview Below)
-```json
-{
-    "models-dir": "path\\to\\folder\\PDF-Extract-Kit-1___0\\models",
-    "layoutreader-model-dir": "path\\to\\folder\\layoutreader",
-    "device-mode": "cpu",
-    ...
-}
-```
-
-##  3. Installing Basic Dependencies for datamax
-1. Clone the repository to your local machine:
-   ```bash
-   git clone <repository-url>
-   ```
-2. Install dependencies into conda:
-   ```bash
-   cd datamax
-   pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
-   ```
-
-
-## Features
-- **Multi-format Support**: Capable of handling various text file types such as PDF, HTML, DOCX, and TXT.
-- **Content Extraction**: Provides powerful content extraction capabilities to accurately retrieve information from complex document structures.
-- **Data Conversion**: Supports converting processed data into markdown format for further analysis.
-- **Batch Processing**: Can handle multiple files at once, improving work efficiency.
-- **Customizable Configuration**: Users can adjust processing parameters according to their needs to meet different business requirements.
-- **Cross-platform Compatibility**: This SDK can run on multiple operating systems, including Windows, MacOS, and Linux.
-
-
-## Technology Stack
-
-- **Programming Language**: Python >= 3.10  
-- **Dependency Libraries**:  
-  - PyMuPDF: For PDF file parsing.  
-  - BeautifulSoup: For HTML file parsing.  
-  - python-docx: For DOCX file parsing.  
-  - pandas: For data processing and conversion.  
-  - paddleocr: For parsing scanned PDFs, tables, and images.  
-- **Development Environment**: Visual Studio Code or PyCharm  
-- **Version Control**: Git  
-
-## Usage Instructions
-### Installing the SDK
-- **Installation Commands**:
-  ```bash
-  ## Local Installation
-  python setup.py sdist bdist_wheel
-  pip install dist/datamax-0.1.3-py3-none-any.whl
-  
-  ## Pip Installation
-  pip install pydatamax
-  ```
-  
-
-- **Importing the Code**:
-    ```python
-    # File Parsing
-    from datamax import DataMax
-    
-    ## Handling a Single File in Two Ways
-    # 1. Using a List of Length 1
-    data = DataMax(file_path=[r"docx_files_example/船视宝概述.doc"])
-    data = data.get_data()
-    
-    # 2. Using a String
-    data = DataMax(file_path=r"docx_files_example/船视宝概述.doc")
-    data = data.get_data()
-    
-    ## Handling Multiple Files
-    # 1. Using a List of Length n
-    data = DataMax(file_path=[r"docx_files_example/船视宝概述1.doc", r"docx_files_example/船视宝概述2.doc"])
-    data = data.get_data()
-    
-    # 2. Passing a Folder Path as a String
-    data = DataMax(file_path=r"docx_files_example/")
-    data = data.get_data()
-    
-    # Data Cleaning
-    """
-    Cleaning rules can be found in datamax/utils/data_cleaner.py
-    abnormal: Abnormal cleaning
-    private: Privacy processing
-    filter: Text filtering
-    """
-    # Direct Use: Clean the text parameter directly and return a string
-    dm = DataMax()
-    data = dm.clean_data(method_list=["abnormal", "private"], text="<div></div>你好 18717777777 \n\n\n\n")
-    
-    # Process Use: Use after get_data() to return the complete data structure
-    dm = DataMax(file_path=r"C:\Users\cykro\Desktop\数据库开发手册.pdf", use_ocr=True)
-    data2 = dm.get_data()
-    cleaned_data = dm.clean_data(method_list=["abnormal", "filter", "private"])
-    
-    # Large Model Pre-annotation Supporting any model that can be called via OpenAI SDK
-    data = DataMax(file_path=r"path\to\xxx.docx")
-    parsed_data = data.get_data()
-    # If no custom messages are passed, the default messages in the SDK will be used
-    messages = [
-        {'role': 'system', 'content': 'You are a helpful assistant.'},
-        {'role': 'user', 'content': 'Who are you?'}
-    ]
-    qa_datas = data.get_pre_label(
-        api_key="sk-xxx",
-        base_url="https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions",
-        model_name="qwen-max",
-        chunk_size=500,
-        chunk_overlap=100,
-        question_number=5,
-        max_workers=5,
-        # message=[]
-    )
-    print(f'Annotated result:{qa_datas}')
-    ```
-
-
-## Examples
-    ```python
-    ## docx | doc | epub | html | txt | ppt | pptx | xls | xlsx
-    from datamax import DataMax
-    data = DataMax(file_path=r"docx_files_example/船视宝概述.doc", to_markdown=True)
-    """
-    Parameters: 
-    file_path: Relative file path / Absolute file path
-    to_markdown: Whether to convert to markdown (default value False, directly returns text) This parameter only supports word files (doc | docx)
-    """
-    
-    ## jpg | jpeg | png | ...(image types)
-    data = DataMax(file_path=r"image.jpg", use_mineru=True)
-    """
-    Parameters:
-    file_path: Relative file path / Absolute file path
-    use_mineru: Whether to use MinerU enhancement
-    """
-    
-    ## pdf
-    from datamax import DataMax
-    data = DataMax(file_path=r"docx_files_example/船视宝概述.pdf", use_mineru=True)
-    """
-    Parameters: 
-    file_path: Relative file path / Absolute file path
-    use_mineru: Whether to use MinerU enhancement
-    """
-    ```
-
-## Contribution Guide
-We welcome any form of contribution, whether it is reporting bugs, suggesting new features, or submitting code improvements. Please read our Contributor's Guide to learn how to get started.
-## License
-This project is licensed under the MIT License. For more details, see the LICENSE file.
-
-## Contact Information
-If you encounter any issues during use, or have any suggestions or feedback, please contact us through the following means:
-- Email: cy.kron@foxmail.com | zhibaohe@hotmail.com
-- Project Homepage: GitHub Project Link
-