pydatamax 0.1.13__py3-none-any.whl → 0.1.14__py3-none-any.whl

datamax/loader/core.py

@@ -1,7 +1,7 @@
 import os
 from typing import List
-from datamax.loader.MinioHandler import MinIOClient
-from datamax.loader.OssHandler import OssClient
+from datamax.loader.minio_handler import MinIOClient
+from datamax.loader.oss_handler import OssClient
 
 
 class DataLoader:

datamax/parser/doc_parser.py

@@ -1,14 +1,14 @@
+import logging
 import os
 import shutil
 import subprocess
 import tempfile
-import chardet
-import logging
 from pathlib import Path
 from typing import Union
-from datamax.parser.base import BaseLife
-from datamax.parser.base import MarkdownOutputVo
 
+import chardet
+
+from datamax.parser.base import BaseLife, MarkdownOutputVo
 
 # 配置日志
 logger = logging.getLogger(__name__)
@@ -24,37 +24,41 @@ class DocParser(BaseLife):
     def doc_to_txt(self, doc_path: str, dir_path: str) -> str:
         """将.doc文件转换为.txt文件"""
         logger.info(f"🔄 开始转换DOC文件为TXT - 源文件: {doc_path}, 输出目录: {dir_path}")
-        
+
         try:
             cmd = f'soffice --headless --convert-to txt "{doc_path}" --outdir "{dir_path}"'
             logger.debug(f"⚡ 执行转换命令: {cmd}")
-            
-            process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+            process = subprocess.Popen(
+                cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            )
             stdout, stderr = process.communicate()
             exit_code = process.returncode
-            
+
             if exit_code == 0:
                 logger.info(f"✅ DOC到TXT转换成功 - 退出码: {exit_code}")
                 if stdout:
                     logger.debug(f"📄 转换输出: {stdout.decode('utf-8', errors='replace')}")
             else:
-                encoding = chardet.detect(stderr)['encoding']
+                encoding = chardet.detect(stderr)["encoding"]
                 if encoding is None:
-                    encoding = 'utf-8'
-                error_msg = stderr.decode(encoding, errors='replace')
+                    encoding = "utf-8"
+                error_msg = stderr.decode(encoding, errors="replace")
                 logger.error(f"❌ DOC到TXT转换失败 - 退出码: {exit_code}, 错误信息: {error_msg}")
-                raise Exception(f"Error Output (detected encoding: {encoding}): {error_msg}")
-            
+                raise Exception(
+                    f"Error Output (detected encoding: {encoding}): {error_msg}"
+                )
+
             fname = str(Path(doc_path).stem)
-            txt_path = os.path.join(dir_path, f'{fname}.txt')
-            
+            txt_path = os.path.join(dir_path, f"{fname}.txt")
+
             if not os.path.exists(txt_path):
                 logger.error(f"❌ 转换后的TXT文件不存在: {txt_path}")
                 raise Exception(f"文件转换失败 {doc_path} ==> {txt_path}")
             else:
                 logger.info(f"🎉 TXT文件转换成功，文件路径: {txt_path}")
                 return txt_path
-                
+
         except subprocess.SubprocessError as e:
             logger.error(f"💥 subprocess执行失败: {str(e)}")
             raise Exception(f"执行转换命令时发生错误: {str(e)}")
@@ -65,25 +69,25 @@ class DocParser(BaseLife):
     def read_txt_file(self, txt_path: str) -> str:
         """读取txt文件内容"""
         logger.info(f"📖 开始读取TXT文件: {txt_path}")
-        
+
         try:
             # 检测文件编码
-            with open(txt_path, 'rb') as f:
+            with open(txt_path, "rb") as f:
                 raw_data = f.read()
-                encoding = chardet.detect(raw_data)['encoding']
+                encoding = chardet.detect(raw_data)["encoding"]
                 if encoding is None:
-                    encoding = 'utf-8'
+                    encoding = "utf-8"
                 logger.debug(f"🔍 检测到文件编码: {encoding}")
-            
+
             # 读取文件内容
-            with open(txt_path, 'r', encoding=encoding, errors='replace') as f:
+            with open(txt_path, "r", encoding=encoding, errors="replace") as f:
                 content = f.read()
-            
+
             logger.info(f"📄 TXT文件读取完成 - 内容长度: {len(content)} 字符")
             logger.debug(f"👀 前100字符预览: {content[:100]}...")
-            
+
             return content
-            
+
         except FileNotFoundError as e:
             logger.error(f"🚫 TXT文件未找到: {str(e)}")
             raise Exception(f"文件未找到: {txt_path}")
@@ -94,27 +98,27 @@ class DocParser(BaseLife):
     def read_doc_file(self, doc_path: str) -> str:
         """读取doc文件并转换为文本"""
         logger.info(f"📖 开始读取DOC文件 - 文件: {doc_path}")
-        
+
         try:
             with tempfile.TemporaryDirectory() as temp_path:
                 logger.debug(f"📁 创建临时目录: {temp_path}")
-                
+
                 temp_dir = Path(temp_path)
-                
+
                 file_path = temp_dir / "tmp.doc"
                 shutil.copy(doc_path, file_path)
                 logger.debug(f"📋 复制文件到临时目录: {doc_path} -> {file_path}")
-                
+
                 # 转换DOC为TXT
                 txt_file_path = self.doc_to_txt(str(file_path), str(temp_path))
                 logger.info(f"🎯 DOC转TXT完成: {txt_file_path}")
-                
+
                 # 读取TXT文件内容
                 content = self.read_txt_file(txt_file_path)
                 logger.info(f"✨ TXT文件内容读取完成，内容长度: {len(content)} 字符")
-                
+
                 return content
-                    
+
         except FileNotFoundError as e:
             logger.error(f"🚫 文件未找到: {str(e)}")
             raise Exception(f"文件未找到: {doc_path}")
@@ -128,24 +132,24 @@ class DocParser(BaseLife):
     def parse(self, file_path: str):
         """解析DOC文件"""
         logger.info(f"🎬 开始解析DOC文件: {file_path}")
-        
+
         try:
             # 验证文件存在
             if not os.path.exists(file_path):
                 logger.error(f"🚫 文件不存在: {file_path}")
                 raise FileNotFoundError(f"文件不存在: {file_path}")
-            
+
             # 验证文件大小
             file_size = os.path.getsize(file_path)
             logger.info(f"📏 文件大小: {file_size} 字节")
-            
+
             title = self.get_file_extension(file_path)
             logger.debug(f"🏷️ 提取文件标题: {title}")
-            
+
             # 使用soffice转换为txt后读取内容
             logger.info("📝 使用soffice转换DOC为TXT并读取内容")
             content = self.read_doc_file(doc_path=file_path)
-            
+
             # 根据to_markdown参数决定是否保持原格式还是处理为markdown格式
             if self.to_markdown:
                 # 简单的文本到markdown转换（保持段落结构）
@@ -154,22 +158,26 @@ class DocParser(BaseLife):
             else:
                 mk_content = content
                 logger.info("📝 保持原始文本格式")
-            
+
             logger.info(f"🎊 文件内容解析完成，最终内容长度: {len(mk_content)} 字符")
-            
-            lifecycle = self.generate_lifecycle(source_file=file_path, domain="Technology",
-                                                usage_purpose="Documentation", life_type="LLM_ORIGIN")
+
+            lifecycle = self.generate_lifecycle(
+                source_file=file_path,
+                domain="Technology",
+                usage_purpose="Documentation",
+                life_type="LLM_ORIGIN",
+            )
             logger.debug("⚙️ 生成lifecycle信息完成")
-            
+
             output_vo = MarkdownOutputVo(title, mk_content)
             output_vo.add_lifecycle(lifecycle)
-            
+
             result = output_vo.to_dict()
             logger.info(f"🏆 DOC文件解析完成: {file_path}")
             logger.debug(f"🔑 返回结果键: {list(result.keys())}")
-            
+
             return result
-            
+
         except Exception as e:
             logger.error(f"💀 解析DOC文件失败: {file_path}, 错误: {str(e)}")
             raise
@@ -178,18 +186,18 @@ class DocParser(BaseLife):
         """将纯文本格式化为简单的markdown格式"""
         if not content.strip():
             return content
-        
-        lines = content.split('\n')
+
+        lines = content.split("\n")
         formatted_lines = []
-        
+
         for line in lines:
             line = line.strip()
             if not line:
-                formatted_lines.append('')
+                formatted_lines.append("")
                 continue
-            
+
             # 简单的markdown格式化规则
             # 可以根据需要扩展更多规则
             formatted_lines.append(line)
-        
-        return '\n'.join(formatted_lines)
+
+        return "\n".join(formatted_lines)

datamax/parser/docx_parser.py

@@ -1,14 +1,14 @@
+import logging
 import os
 import shutil
 import subprocess
 import tempfile
-import chardet
-import logging
 from pathlib import Path
 from typing import Union
-from datamax.parser.base import BaseLife
-from datamax.parser.base import MarkdownOutputVo
 
+import chardet
+
+from datamax.parser.base import BaseLife, MarkdownOutputVo
 
 # 配置日志
 logger = logging.getLogger(__name__)
@@ -19,42 +19,48 @@ class DocxParser(BaseLife):
         super().__init__()
         self.file_path = file_path
         self.to_markdown = to_markdown
-        logger.info(f"🚀 DocxParser初始化完成 - 文件路径: {file_path}, 转换为markdown: {to_markdown}")
+        logger.info(
+            f"🚀 DocxParser初始化完成 - 文件路径: {file_path}, 转换为markdown: {to_markdown}"
+        )
 
     def docx_to_txt(self, docx_path: str, dir_path: str) -> str:
         """将.docx文件转换为.txt文件"""
         logger.info(f"🔄 开始转换DOCX文件为TXT - 源文件: {docx_path}, 输出目录: {dir_path}")
-        
+
         try:
             cmd = f'soffice --headless --convert-to txt "{docx_path}" --outdir "{dir_path}"'
             logger.debug(f"⚡ 执行转换命令: {cmd}")
-            
-            process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+            process = subprocess.Popen(
+                cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            )
             stdout, stderr = process.communicate()
             exit_code = process.returncode
-            
+
             if exit_code == 0:
                 logger.info(f"✅ DOCX到TXT转换成功 - 退出码: {exit_code}")
                 if stdout:
                     logger.debug(f"📄 转换输出: {stdout.decode('utf-8', errors='replace')}")
             else:
-                encoding = chardet.detect(stderr)['encoding']
+                encoding = chardet.detect(stderr)["encoding"]
                 if encoding is None:
-                    encoding = 'utf-8'
-                error_msg = stderr.decode(encoding, errors='replace')
+                    encoding = "utf-8"
+                error_msg = stderr.decode(encoding, errors="replace")
                 logger.error(f"❌ DOCX到TXT转换失败 - 退出码: {exit_code}, 错误信息: {error_msg}")
-                raise Exception(f"Error Output (detected encoding: {encoding}): {error_msg}")
-            
+                raise Exception(
+                    f"Error Output (detected encoding: {encoding}): {error_msg}"
+                )
+
             fname = str(Path(docx_path).stem)
-            txt_path = os.path.join(dir_path, f'{fname}.txt')
-            
+            txt_path = os.path.join(dir_path, f"{fname}.txt")
+
             if not os.path.exists(txt_path):
                 logger.error(f"❌ 转换后的TXT文件不存在: {txt_path}")
                 raise Exception(f"文件转换失败 {docx_path} ==> {txt_path}")
             else:
                 logger.info(f"🎉 TXT文件转换成功，文件路径: {txt_path}")
                 return txt_path
-                
+
         except subprocess.SubprocessError as e:
             logger.error(f"💥 subprocess执行失败: {str(e)}")
             raise Exception(f"执行转换命令时发生错误: {str(e)}")
@@ -65,25 +71,25 @@ class DocxParser(BaseLife):
     def read_txt_file(self, txt_path: str) -> str:
         """读取txt文件内容"""
         logger.info(f"📖 开始读取TXT文件: {txt_path}")
-        
+
         try:
             # 检测文件编码
-            with open(txt_path, 'rb') as f:
+            with open(txt_path, "rb") as f:
                 raw_data = f.read()
-                encoding = chardet.detect(raw_data)['encoding']
+                encoding = chardet.detect(raw_data)["encoding"]
                 if encoding is None:
-                    encoding = 'utf-8'
+                    encoding = "utf-8"
                 logger.debug(f"🔍 检测到文件编码: {encoding}")
-            
+
             # 读取文件内容
-            with open(txt_path, 'r', encoding=encoding, errors='replace') as f:
+            with open(txt_path, "r", encoding=encoding, errors="replace") as f:
                 content = f.read()
-            
+
             logger.info(f"📄 TXT文件读取完成 - 内容长度: {len(content)} 字符")
             logger.debug(f"👀 前100字符预览: {content[:100]}...")
-            
+
             return content
-            
+
         except FileNotFoundError as e:
             logger.error(f"🚫 TXT文件未找到: {str(e)}")
             raise Exception(f"文件未找到: {txt_path}")
@@ -94,27 +100,27 @@ class DocxParser(BaseLife):
     def read_docx_file(self, docx_path: str) -> str:
         """读取docx文件并转换为文本"""
         logger.info(f"📖 开始读取DOCX文件 - 文件: {docx_path}")
-        
+
         try:
             with tempfile.TemporaryDirectory() as temp_path:
                 logger.debug(f"📁 创建临时目录: {temp_path}")
-                
+
                 temp_dir = Path(temp_path)
-                
+
                 file_path = temp_dir / "tmp.docx"
                 shutil.copy(docx_path, file_path)
                 logger.debug(f"📋 复制文件到临时目录: {docx_path} -> {file_path}")
-                
+
                 # 转换DOCX为TXT
                 txt_file_path = self.docx_to_txt(str(file_path), str(temp_path))
                 logger.info(f"🎯 DOCX转TXT完成: {txt_file_path}")
-                
+
                 # 读取TXT文件内容
                 content = self.read_txt_file(txt_file_path)
                 logger.info(f"✨ TXT文件内容读取完成，内容长度: {len(content)} 字符")
-                
+
                 return content
-                    
+
         except FileNotFoundError as e:
             logger.error(f"🚫 文件未找到: {str(e)}")
             raise Exception(f"文件未找到: {docx_path}")
@@ -128,31 +134,31 @@ class DocxParser(BaseLife):
     def parse(self, file_path: str):
         """解析DOCX文件"""
         logger.info(f"🎬 开始解析DOCX文件: {file_path}")
-        
+
         try:
             # 验证文件存在
             if not os.path.exists(file_path):
                 logger.error(f"🚫 文件不存在: {file_path}")
                 raise FileNotFoundError(f"文件不存在: {file_path}")
-            
+
             # 验证文件扩展名
-            if not file_path.lower().endswith('.docx'):
+            if not file_path.lower().endswith(".docx"):
                 logger.warning(f"⚠️ 文件扩展名不是.docx: {file_path}")
-            
+
             # 验证文件大小
             file_size = os.path.getsize(file_path)
             logger.info(f"📏 文件大小: {file_size} 字节")
-            
+
             if file_size == 0:
                 logger.warning(f"⚠️ 文件大小为0字节: {file_path}")
-            
+
             title = self.get_file_extension(file_path)
             logger.debug(f"🏷️ 提取文件标题: {title}")
-            
+
             # 使用soffice转换为txt后读取内容
             logger.info("📝 使用soffice转换DOCX为TXT并读取内容")
             content = self.read_docx_file(docx_path=file_path)
-            
+
             # 根据to_markdown参数决定是否保持原格式还是处理为markdown格式
             if self.to_markdown:
                 # 简单的文本到markdown转换（保持段落结构）
@@ -161,26 +167,30 @@ class DocxParser(BaseLife):
             else:
                 mk_content = content
                 logger.info("📝 保持原始文本格式")
-            
+
             logger.info(f"🎊 文件内容解析完成，最终内容长度: {len(mk_content)} 字符")
-            
+
             # 检查内容是否为空
             if not mk_content.strip():
                 logger.warning(f"⚠️ 解析出的内容为空: {file_path}")
-            
-            lifecycle = self.generate_lifecycle(source_file=file_path, domain="Technology",
-                                                usage_purpose="Documentation", life_type="LLM_ORIGIN")
+
+            lifecycle = self.generate_lifecycle(
+                source_file=file_path,
+                domain="Technology",
+                usage_purpose="Documentation",
+                life_type="LLM_ORIGIN",
+            )
             logger.debug("⚙️ 生成lifecycle信息完成")
-            
+
             output_vo = MarkdownOutputVo(title, mk_content)
             output_vo.add_lifecycle(lifecycle)
-            
+
             result = output_vo.to_dict()
             logger.info(f"🏆 DOCX文件解析完成: {file_path}")
             logger.debug(f"🔑 返回结果键: {list(result.keys())}")
-            
+
             return result
-            
+
         except FileNotFoundError as e:
             logger.error(f"🚫 文件不存在错误: {str(e)}")
             raise
@@ -188,25 +198,27 @@ class DocxParser(BaseLife):
             logger.error(f"🔒 文件权限错误: {str(e)}")
             raise Exception(f"无权限访问文件: {file_path}")
         except Exception as e:
-            logger.error(f"💀 解析DOCX文件失败: {file_path}, 错误类型: {type(e).__name__}, 错误信息: {str(e)}")
+            logger.error(
+                f"💀 解析DOCX文件失败: {file_path}, 错误类型: {type(e).__name__}, 错误信息: {str(e)}"
+            )
             raise
 
     def format_as_markdown(self, content: str) -> str:
         """将纯文本格式化为简单的markdown格式"""
         if not content.strip():
             return content
-        
-        lines = content.split('\n')
+
+        lines = content.split("\n")
         formatted_lines = []
-        
+
         for line in lines:
             line = line.strip()
             if not line:
-                formatted_lines.append('')
+                formatted_lines.append("")
                 continue
-            
+
             # 简单的markdown格式化规则
             # 可以根据需要扩展更多规则
             formatted_lines.append(line)
-        
-        return '\n'.join(formatted_lines)
+
+        return "\n".join(formatted_lines)

datamax/parser/xlsx_parser.py

@@ -1,12 +1,13 @@
-import multiprocessing
-import time
 import logging
+import multiprocessing
 import os
+import time
+import warnings
 from multiprocessing import Queue
-from datamax.parser.base import MarkdownOutputVo
-from datamax.parser.base import BaseLife
+
 import pandas as pd
-import warnings
+
+from datamax.parser.base import BaseLife, MarkdownOutputVo
 
 warnings.filterwarnings("ignore")
 
@@ -26,42 +27,44 @@ class XlsxParser(BaseLife):
     def _parse_with_pandas(self, file_path: str) -> str:
         """使用pandas读取Excel并转换为markdown"""
         logger.info(f"🐼 开始使用pandas读取Excel文件: {file_path}")
-        
+
         try:
             # 验证文件存在
             if not os.path.exists(file_path):
                 logger.error(f"🚫 Excel文件不存在: {file_path}")
                 raise FileNotFoundError(f"文件不存在: {file_path}")
-            
+
             # 验证文件大小
             file_size = os.path.getsize(file_path)
             logger.info(f"📏 文件大小: {file_size} 字节")
-            
+
             if file_size == 0:
                 logger.warning(f"⚠️ 文件大小为0字节: {file_path}")
                 return "*文件为空*"
-            
+
             # 使用pandas读取Excel文件
             logger.debug("📊 正在读取Excel数据...")
             df = pd.read_excel(file_path, sheet_name=None)  # 读取所有工作表
-            
+
             markdown_content = ""
-            
+
             if isinstance(df, dict):
                 # 多个工作表
                 logger.info(f"📑 检测到多个工作表，共 {len(df)} 个")
                 for sheet_name, sheet_df in df.items():
                     logger.debug(f"📋 处理工作表: {sheet_name}, 形状: {sheet_df.shape}")
                     markdown_content += f"## 工作表: {sheet_name}\n\n"
-                    
+
                     if not sheet_df.empty:
                         # 清理数据：移除完全为空的行和列
-                        sheet_df = sheet_df.dropna(how='all').dropna(axis=1, how='all')
-                        
+                        sheet_df = sheet_df.dropna(how="all").dropna(axis=1, how="all")
+
                         if not sheet_df.empty:
                             sheet_markdown = sheet_df.to_markdown(index=False)
                             markdown_content += sheet_markdown + "\n\n"
-                            logger.debug(f"✅ 工作表 {sheet_name} 转换完成，有效数据形状: {sheet_df.shape}")
+                            logger.debug(
+                                f"✅ 工作表 {sheet_name} 转换完成，有效数据形状: {sheet_df.shape}"
+                            )
                         else:
                             markdown_content += "*该工作表无有效数据*\n\n"
                             logger.warning(f"⚠️ 工作表 {sheet_name} 清理后无有效数据")
@@ -73,8 +76,8 @@ class XlsxParser(BaseLife):
                 logger.info(f"📄 单个工作表，形状: {df.shape}")
                 if not df.empty:
                     # 清理数据：移除完全为空的行和列
-                    df = df.dropna(how='all').dropna(axis=1, how='all')
-                    
+                    df = df.dropna(how="all").dropna(axis=1, how="all")
+
                     if not df.empty:
                         markdown_content = df.to_markdown(index=False)
                         logger.info(f"✅ 工作表转换完成，有效数据形状: {df.shape}")
@@ -84,12 +87,12 @@ class XlsxParser(BaseLife):
                 else:
                     markdown_content = "*工作表为空*"
                     logger.warning("⚠️ 工作表为空")
-            
+
             logger.info(f"🎊 pandas转换完成，markdown内容长度: {len(markdown_content)} 字符")
             logger.debug(f"👀 前200字符预览: {markdown_content[:200]}...")
-            
+
             return markdown_content
-            
+
         except FileNotFoundError as e:
             logger.error(f"🚫 文件未找到: {str(e)}")
             raise
@@ -106,41 +109,41 @@ class XlsxParser(BaseLife):
     def _parse(self, file_path: str, result_queue: Queue) -> dict:
         """解析Excel文件的核心方法"""
         logger.info(f"🎬 开始解析Excel文件: {file_path}")
-        
+
         try:
             # 使用pandas解析Excel
             logger.info("🐼 使用pandas模式解析Excel")
             mk_content = self._parse_with_pandas(file_path)
-            
+
             # 检查内容是否为空
             if not mk_content.strip():
                 logger.warning(f"⚠️ 解析出的内容为空: {file_path}")
                 mk_content = "*无法解析文件内容*"
-            
+
             logger.info(f"🎊 文件内容解析完成，最终内容长度: {len(mk_content)} 字符")
-            
+
             # 生成lifecycle信息
             lifecycle = self.generate_lifecycle(
                 source_file=file_path,
                 domain="Technology",
                 usage_purpose="Documentation",
-                life_type="LLM_ORIGIN"
+                life_type="LLM_ORIGIN",
             )
             logger.debug("⚙️ 生成lifecycle信息完成")
-            
+
             # 创建输出对象
             title = self.get_file_extension(file_path)
             output_vo = MarkdownOutputVo(title, mk_content)
             output_vo.add_lifecycle(lifecycle)
-            
+
             result = output_vo.to_dict()
             result_queue.put(result)
             logger.info(f"🏆 Excel文件解析完成: {file_path}")
             logger.debug(f"🔑 返回结果键: {list(result.keys())}")
-            
+
             time.sleep(0.5)  # 给队列一点时间
             return result
-            
+
         except Exception as e:
             logger.error(f"💀 解析Excel文件失败: {file_path}, 错误: {str(e)}")
             # 将错误也放入队列
@@ -151,58 +154,62 @@ class XlsxParser(BaseLife):
     def parse(self, file_path: str) -> dict:
         """解析Excel文件 - 支持多进程和超时控制"""
         logger.info(f"🚀 启动Excel解析进程 - 文件: {file_path}, 超时: {self.timeout}s")
-        
+
         try:
             # 验证文件存在
             if not os.path.exists(file_path):
                 logger.error(f"🚫 文件不存在: {file_path}")
                 raise FileNotFoundError(f"文件不存在: {file_path}")
-            
+
             # 验证文件扩展名
-            if not file_path.lower().endswith(('.xlsx', '.xls')):
+            if not file_path.lower().endswith((".xlsx", ".xls")):
                 logger.warning(f"⚠️ 文件扩展名不是Excel格式: {file_path}")
-            
+
             result_queue = Queue()
-            process = multiprocessing.Process(target=self._parse, args=(file_path, result_queue))
+            process = multiprocessing.Process(
+                target=self._parse, args=(file_path, result_queue)
+            )
             process.start()
             logger.debug(f"⚡ 启动子进程，PID: {process.pid}")
-            
+
             start_time = time.time()
-            
+
             # 等待解析完成或超时
             while time.time() - start_time < self.timeout:
                 elapsed_time = int(time.time() - start_time)
                 logger.debug(f"⏱️ 等待解析完成... {elapsed_time}s")
-                
+
                 if not process.is_alive():
                     logger.debug("✅ 子进程已完成")
                     break
-                    
+
                 if not result_queue.empty():
                     result = result_queue.get()
                     process.join()  # 等待进程正常结束
-                    
+
                     # 检查是否是错误结果
                     if "error" in result:
                         logger.error(f"💥 子进程返回错误: {result['error']}")
                         raise Exception(result["error"])
-                    
+
                     logger.info(f"🎉 Excel解析成功完成，耗时: {elapsed_time}s")
                     return result
-                    
+
                 time.sleep(1)
             else:
                 # 超时处理
                 logger.error(f"⏰ 解析超时 ({self.timeout}s)，终止进程")
                 process.terminate()
                 process.join(timeout=5)  # 给进程5秒时间优雅退出
-                
+
                 if process.is_alive():
                     logger.error("💀 强制杀死进程")
                     process.kill()
-                    
+
                 raise TimeoutError(f"Excel解析超时: {file_path}")
-                
+
         except Exception as e:
-            logger.error(f"💀 Excel解析失败: {file_path}, 错误类型: {type(e).__name__}, 错误信息: {str(e)}")
-            raise
+            logger.error(
+                f"💀 Excel解析失败: {file_path}, 错误类型: {type(e).__name__}, 错误信息: {str(e)}"
+            )
+            raise

pydatamax-0.1.14.dist-info/METADATA

@@ -0,0 +1,228 @@
+Metadata-Version: 2.4
+Name: pydatamax
+Version: 0.1.14
+Summary: A library for parsing and converting various file formats.
+Home-page: https://github.com/Hi-Dolphin/datamax
+Author: ccy
+Author-email: cy.kron@foxmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: oss2<3.0.0,>=2.19.1
+Requires-Dist: aliyun-python-sdk-core<3.0.0,>=2.16.0
+Requires-Dist: aliyun-python-sdk-kms<3.0.0,>=2.16.5
+Requires-Dist: crcmod<2.0.0,>=1.7
+Requires-Dist: langdetect<2.0.0,>=1.0.9
+Requires-Dist: loguru<1.0.0,>=0.7.3
+Requires-Dist: python-docx<2.0.0,>=1.1.2
+Requires-Dist: python-dotenv<2.0.0,>=1.1.0
+Requires-Dist: pymupdf<2.0.0,>=1.26.0
+Requires-Dist: pypdf<6.0.0,>=5.5.0
+Requires-Dist: openpyxl<4.0.0,>=3.1.5
+Requires-Dist: pandas<3.0.0,>=2.2.3
+Requires-Dist: numpy<3.0.0,>=2.2.6
+Requires-Dist: requests<3.0.0,>=2.32.3
+Requires-Dist: tqdm<5.0.0,>=4.67.1
+Requires-Dist: pydantic<3.0.0,>=2.11.5
+Requires-Dist: pydantic-settings<3.0.0,>=2.9.1
+Requires-Dist: python-magic<1.0.0,>=0.4.27
+Requires-Dist: PyYAML<7.0.0,>=6.0.2
+Requires-Dist: Pillow<12.0.0,>=11.2.1
+Requires-Dist: packaging<25.0,>=24.2
+Requires-Dist: beautifulsoup4<5.0.0,>=4.13.4
+Requires-Dist: minio<8.0.0,>=7.2.15
+Requires-Dist: openai<2.0.0,>=1.82.0
+Requires-Dist: jionlp<2.0.0,>=1.5.23
+Requires-Dist: chardet<6.0.0,>=5.2.0
+Requires-Dist: python-pptx<2.0.0,>=1.0.2
+Requires-Dist: tiktoken<1.0.0,>=0.9.0
+Requires-Dist: markitdown<1.0.0,>=0.1.1
+Requires-Dist: xlrd<3.0.0,>=2.0.1
+Requires-Dist: tabulate<1.0.0,>=0.9.0
+Requires-Dist: unstructured<1.0.0,>=0.17.2
+Requires-Dist: markdown<4.0.0,>=3.8
+Requires-Dist: langchain<1.0.0,>=0.3.0
+Requires-Dist: langchain-community<1.0.0,>=0.3.0
+Requires-Dist: ebooklib==0.19
+Requires-Dist: setuptools
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# DataMax
+
+<div align="center">
+
+[中文](README_zh.md) | **English**
+
+[![PyPI version](https://badge.fury.io/py/pydatamax.svg)](https://badge.fury.io/py/pydatamax) [![Python](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+</div>
+
+A powerful multi-format file parsing, data cleaning, and AI annotation toolkit.
+
+## ✨ Core Features
+
+- 🔄 **Multi-format Support**: PDF, DOCX/DOC, PPT/PPTX, XLS/XLSX, HTML, EPUB, TXT, images, and more
+- 🧹 **Intelligent Cleaning**: Three-layer cleaning process with anomaly detection, privacy protection, and text filtering
+- 🤖 **AI Annotation**: LLM-based automatic data annotation and pre-labeling
+- ⚡ **Batch Processing**: Efficient multi-file parallel processing
+- 🎯 **Easy Integration**: Clean API design, ready to use out of the box
+
+## 🚀 Quick Start
+
+### Installation
+
+```bash
+pip install pydatamax
+```
+
+### Basic Usage
+
+```python
+from datamax import DataMax
+
+# Parse a single file
+dm = DataMax(file_path="document.pdf")
+data = dm.get_data()
+
+# Batch processing
+dm = DataMax(file_path=["file1.docx", "file2.pdf"])
+data = dm.get_data()
+
+# Data cleaning
+cleaned_data = dm.clean_data(method_list=["abnormal", "private", "filter"])
+
+# AI annotation
+qa_data = dm.get_pre_label(
+    api_key="your-api-key",
+    base_url="https://api.openai.com/v1",
+    model_name="gpt-3.5-turbo"
+)
+```
+
+## 📖 Detailed Documentation
+
+### File Parsing
+
+#### Supported Formats
+
+| Format | Extensions | Special Features |
+|--------|------------|------------------|
+| Documents | `.pdf`, `.docx`, `.doc` | OCR support, Markdown conversion |
+| Spreadsheets | `.xlsx`, `.xls` | Structured data extraction |
+| Presentations | `.pptx`, `.ppt` | Slide content extraction |
+| Web | `.html`, `.epub` | Tag parsing |
+| Images | `.jpg`, `.png`, `.jpeg` | OCR text recognition |
+| Text | `.txt` | Automatic encoding detection |
+
+#### Advanced Features
+
+```python
+# Advanced PDF parsing (requires MinerU)
+dm = DataMax(file_path="complex.pdf", use_mineru=True)
+
+# Word to Markdown conversion
+dm = DataMax(file_path="document.docx", to_markdown=True)
+
+# Image OCR
+dm = DataMax(file_path="image.jpg", use_ocr=True)
+```
+
+### Data Cleaning
+
+```python
+# Three cleaning modes
+dm.clean_data(method_list=[
+    "abnormal",  # Anomaly data processing
+    "private",   # Privacy information masking
+    "filter"     # Text filtering and normalization
+])
+```
+
+### AI Annotation
+
+```python
+# Custom annotation tasks
+qa_data = dm.get_pre_label(
+    api_key="sk-xxx",
+    base_url="https://api.provider.com/v1",
+    model_name="model-name",
+    chunk_size=500,        # Text chunk size
+    chunk_overlap=100,     # Overlap length
+    question_number=5,     # Questions per chunk
+    max_workers=5          # Concurrency
+)
+```
+
+## ⚙️ Environment Setup
+
+### Optional Dependencies
+
+#### LibreOffice (DOC file support)
+
+**Ubuntu/Debian:**
+```bash
+sudo apt-get install libreoffice
+```
+
+**Windows:**
+1. Download and install [LibreOffice](https://www.libreoffice.org/download/)
+2. Add to environment variables: `C:\Program Files\LibreOffice\program`
+
+#### MinerU (Advanced PDF parsing)
+
+```bash
+# Create virtual environment
+conda create -n mineru python=3.10
+conda activate mineru
+
+# Install MinerU
+pip install -U "magic-pdf[full]" --extra-index-url https://wheels.myhloli.com
+```
+
+For detailed configuration, please refer to [MinerU Documentation](https://github.com/opendatalab/MinerU)
+
+## 🛠️ Development
+
+### Local Installation
+
+```bash
+git clone https://github.com/Hi-Dolphin/datamax.git
+cd datamax
+pip install -r requirements.txt
+python setup.py install
+```
+
+## 📋 System Requirements
+
+- Python >= 3.10
+- Supports Windows, macOS, Linux
+
+## 🤝 Contributing
+
+Issues and Pull Requests are welcome!
+
+## 📄 License
+
+This project is licensed under the [MIT License](LICENSE).
+
+## 📞 Contact Us
+
+- 📧 Email: cy.kron@foxmail.com
+- 🐛 Issues: [GitHub Issues](https://github.com/Hi-Dolphin/datamax/issues)
+- 📚 Documentation: [Project Homepage](https://github.com/Hi-Dolphin/datamax)
+
+---
+
+⭐ If this project helps you, please give us a star! 

{pydatamax-0.1.13.dist-info → pydatamax-0.1.14.dist-info}/RECORD

@@ -1,14 +1,14 @@
 datamax/__init__.py,sha256=Kbs8ITE6suPy0VL8WzKH8A_iAGqukC0jIHcFGLgoBw8,28
-datamax/loader/MinioHandler.py,sha256=0XLvtKayhI2cjPU7S7m91tSzPcaOC924ByJNbabY6So,6386
-datamax/loader/OssHandler.py,sha256=YFbUfH2vkKRVg8OhkYpOrlYHrIX4zofbHFVe8nPpsQ4,7457
 datamax/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamax/loader/core.py,sha256=tSIkOw5D3EVFYme1b7joFt0e_LxJdf-mdUzxpyVt0VI,5098
+datamax/loader/core.py,sha256=C5VhDXkv0HTRcF6WWtztatUZHURZBD-KoZpKyqXfD7U,5100
+datamax/loader/minio_handler.py,sha256=0XLvtKayhI2cjPU7S7m91tSzPcaOC924ByJNbabY6So,6386
+datamax/loader/oss_handler.py,sha256=YFbUfH2vkKRVg8OhkYpOrlYHrIX4zofbHFVe8nPpsQ4,7457
 datamax/parser/__init__.py,sha256=Jilq2PLBNonmoXKATzsIHWWvFuBdlcV2dbSP1cOZ6zg,111
 datamax/parser/base.py,sha256=riGcMn4m295_qf9O0-NbHU2BcHGBXvoF4T3fWj9vgUQ,2514
 datamax/parser/core.py,sha256=9rzIjsVTRacPTUTAVa5gm5fx0h95LxYnw0lEGqjIIB4,11437
 datamax/parser/csv_parser.py,sha256=IcyVq8mGE6auIcUInXGYWDnh0H0XJ_3SyQrLVRrS7i0,190
-datamax/parser/doc_parser.py,sha256=WIWZqvWT4bbquMn1t5Y4P3rEFG6YZ6z3b-f-5yCEtwU,8266
-datamax/parser/docx_parser.py,sha256=Ipk9ea281N8Edj74tnqUpc_MGZgD4qn780MX_QA9SiU,9111
+datamax/parser/doc_parser.py,sha256=wVNYn7dkPI12pW_YImDgoceLFWS3RvFpzbFQwVlrnNo,7936
+datamax/parser/docx_parser.py,sha256=o9_1VBDc8nBCmsEMv0sKsVcPMxiAxY5pl0mUvOcoOJc,8796
 datamax/parser/epub_parser.py,sha256=ljCGxLBPwE5gXVKARJec93VpP4dE9R2GspzuSZBkqPQ,1557
 datamax/parser/html_parser.py,sha256=xQaaK8674QbQwE-Up9X0DJIH0Gg0mR2KoI7fJ6iw2m0,1393
 datamax/parser/image_parser.py,sha256=qGCndc_21PwsfuxFG03wHSsV0uc-XMBaW3VDbsJQd90,1233
@@ -19,7 +19,7 @@ datamax/parser/ppt_parser.py,sha256=Niu3Ina6I6m6lAMS1Z-A7rUbR_iFGmNTaASBoNH_vZ0,
 datamax/parser/pptx_parser.py,sha256=sFWyOa3QNIs4BgtpmSzFQgsgPmunfGqCqi6fulbLFW0,1811
 datamax/parser/txt_parser.py,sha256=4DIP1LVOw21NDdtqG2RTD_hMcHufkvC8kr048AkuLFs,1682
 datamax/parser/xls_parser.py,sha256=pRlqgg96f76H8UqXQfheQT9O0ThdP7958hKUCEyQfPM,954
-datamax/parser/xlsx_parser.py,sha256=tyLU6wa3F31p7JaoCpML6TJyzYd2Lpeuhzs4036en2U,9274
+datamax/parser/xlsx_parser.py,sha256=Vw6XfoQyu6aQUSIueR-krByMA_WOb5fasf4VmKxjVio,8905
 datamax/utils/__init__.py,sha256=d69SJvqOXzItyg9rEcLc4z67Lw9vACispOe3x7NvZLA,1051
 datamax/utils/constants.py,sha256=A0S56mkIfeT6oQmOd-VGTChzLOSBUqsG4skMmLt6uNk,4507
 datamax/utils/data_cleaner.py,sha256=zlk2dXmhU-_9KVfqmqMGr967v-nc7Iv8ZKRdMkIJsGM,7784
@@ -30,10 +30,10 @@ datamax/utils/paddleocr_pdf_operator.py,sha256=Tnb-5SzUd6OXM-XeaL8vdPnsOhgG_GKz-
 datamax/utils/ppt_extract.py,sha256=nd6KSqEzxANrPhNPUZY4ogAyxHzKCbdsI5ZfDQCz0Cw,6164
 datamax/utils/qa_generator.py,sha256=d75an9JEyT6sxlSjdmWYveQshfyTb0v4aGSuTpTJa0A,12561
 datamax/utils/tokenizer.py,sha256=Y8XB06XQVsNuG8IPl_4iBZj2yu1xzXldVbmZtXFMQM4,859
-pydatamax-0.1.13.dist-info/licenses/LICENSE,sha256=LvCq2Pc7MejIvfNeRl_kAM5l_KXWlQFiC-Sjp7kqFf8,1067
+pydatamax-0.1.14.dist-info/licenses/LICENSE,sha256=LvCq2Pc7MejIvfNeRl_kAM5l_KXWlQFiC-Sjp7kqFf8,1067
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_basic.py,sha256=4AByx25-MIt6_zmzxpFRoSCBqLtIjyfTwFLb1UCJz6k,303
-pydatamax-0.1.13.dist-info/METADATA,sha256=knte2YZ9jdSGxmO0fzBVtMFAcq1exCKyEdfBde4aCjA,9731
-pydatamax-0.1.13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pydatamax-0.1.13.dist-info/top_level.txt,sha256=DvdVIUrxJDgRUYiGA5vznYZIP-K8ZnDkTZfrqYLNZMQ,14
-pydatamax-0.1.13.dist-info/RECORD,,
+pydatamax-0.1.14.dist-info/METADATA,sha256=P-wz8Log3gcUMftKTd2qrcmNuzpp-HOn8gVVe8cTceM,6314
+pydatamax-0.1.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pydatamax-0.1.14.dist-info/top_level.txt,sha256=DvdVIUrxJDgRUYiGA5vznYZIP-K8ZnDkTZfrqYLNZMQ,14
+pydatamax-0.1.14.dist-info/RECORD,,

pydatamax-0.1.13.dist-info/METADATA

@@ -1,280 +0,0 @@
-Metadata-Version: 2.4
-Name: pydatamax
-Version: 0.1.13
-Summary: A library for parsing and converting various file formats.
-Home-page: https://github.com/Hi-Dolphin/datamax
-Author: ccy
-Author-email: cy.kron@foxmail.com
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: oss2<3.0.0,>=2.19.1
-Requires-Dist: aliyun-python-sdk-core<3.0.0,>=2.16.0
-Requires-Dist: aliyun-python-sdk-kms<3.0.0,>=2.16.5
-Requires-Dist: crcmod<2.0.0,>=1.7
-Requires-Dist: langdetect<2.0.0,>=1.0.9
-Requires-Dist: loguru<1.0.0,>=0.7.3
-Requires-Dist: python-docx<2.0.0,>=1.1.2
-Requires-Dist: python-dotenv<2.0.0,>=1.1.0
-Requires-Dist: pymupdf<2.0.0,>=1.26.0
-Requires-Dist: pypdf<6.0.0,>=5.5.0
-Requires-Dist: openpyxl<4.0.0,>=3.1.5
-Requires-Dist: pandas<3.0.0,>=2.2.3
-Requires-Dist: numpy<3.0.0,>=2.2.6
-Requires-Dist: requests<3.0.0,>=2.32.3
-Requires-Dist: tqdm<5.0.0,>=4.67.1
-Requires-Dist: pydantic<3.0.0,>=2.11.5
-Requires-Dist: pydantic-settings<3.0.0,>=2.9.1
-Requires-Dist: python-magic<1.0.0,>=0.4.27
-Requires-Dist: PyYAML<7.0.0,>=6.0.2
-Requires-Dist: Pillow<12.0.0,>=11.2.1
-Requires-Dist: packaging<25.0,>=24.2
-Requires-Dist: beautifulsoup4<5.0.0,>=4.13.4
-Requires-Dist: minio<8.0.0,>=7.2.15
-Requires-Dist: openai<2.0.0,>=1.82.0
-Requires-Dist: jionlp<2.0.0,>=1.5.23
-Requires-Dist: chardet<6.0.0,>=5.2.0
-Requires-Dist: python-pptx<2.0.0,>=1.0.2
-Requires-Dist: tiktoken<1.0.0,>=0.9.0
-Requires-Dist: markitdown<1.0.0,>=0.1.1
-Requires-Dist: xlrd<3.0.0,>=2.0.1
-Requires-Dist: tabulate<1.0.0,>=0.9.0
-Requires-Dist: unstructured<1.0.0,>=0.17.2
-Requires-Dist: markdown<4.0.0,>=3.8
-Requires-Dist: langchain<1.0.0,>=0.3.0
-Requires-Dist: langchain-community<1.0.0,>=0.3.0
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license-file
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary
-
-# DataMax
-
-## Overview
-DataMax is designed as a comprehensive solution for processing diverse file formats, performing data cleaning, and facilitating data annotation.
-
-## Key Features
-
-### File Processing Capabilities
-Currently supports reading, conversion, and extraction from:
-- PDF, HTML  
-- DOCX/DOC, PPT/PPTX  
-- EPUB  
-- Images
-- XLS/XLSX spreadsheets  
-- Plain text (TXT)  
-
-### Data Cleaning Pipeline
-Three-tiered cleaning process:
-1. Anomaly detection and handling  
-2. Privacy protection processing  
-3. Text filtering and normalization  
-
-### AI-Powered Data Annotation
-Implements an LLM+Prompt to:
-- Continuously generate pre-labeled datasets  
-- Provide optimized training data for model fine-tuning  
-
-
-## Installation Guide (Key Dependencies)
-Dependencies include libreoffice, datamax, and MinerU.
-
-### 1. Installing libreoffice Dependency
-**Note:** Without datamax, .doc files will not be supported.
-
-#### Linux (Debian/Ubuntu)
-```bash
-sudo apt-get update
-sudo apt-get install libreoffice
-```
-### Windows
-```text
-Install LibreOffice from: [Download LibreOffice](https://www.libreoffice.org/download/download-libreoffice/?spm=5176.28103460.0.0.5b295d275bpHzh)  
-Add to environment variable: `$env:PATH += ";C:\Program Files\LibreOffice\program"`
-```
-### Checking LibreOffice Installation
-```bash
-soffice --version
-```
-
-## 2. Installing MinerU Dependency
-Note: Without MinerU, advanced OCR parsing for PDFs will not be supported.
-### Create a Virtual Environment and Install Basic Dependencies
-```bash
-conda create -n mineru python=3.10
-conda activate mineru
-pip install -U "magic-pdf[full]" --extra-index-url https://wheels.myhloli.com -i https://mirrors.aliyun.com/pypi/simple
-```
-### Installing Model Weight Files
-https://github.com/opendatalab/MinerU/blob/master/docs/how_to_download_models_zh_cn.md
-```bash
-pip install modelscope
-wget https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/scripts/download_models.py -O download_models.py
-python download_models.py
-```
-
-### Modify the Configuration File magic-pdf.json (Located in the User Directory, Template Preview Below)
-```json
-{
-    "models-dir": "path\\to\\folder\\PDF-Extract-Kit-1___0\\models",
-    "layoutreader-model-dir": "path\\to\\folder\\layoutreader",
-    "device-mode": "cpu",
-    ...
-}
-```
-
-##  3. Installing Basic Dependencies for datamax
-1. Clone the repository to your local machine:
-   ```bash
-   git clone <repository-url>
-   ```
-2. Install dependencies into conda:
-   ```bash
-   cd datamax
-   pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
-   ```
-
-
-## Features
-- **Multi-format Support**: Capable of handling various text file types such as PDF, HTML, DOCX, and TXT.
-- **Content Extraction**: Provides powerful content extraction capabilities to accurately retrieve information from complex document structures.
-- **Data Conversion**: Supports converting processed data into markdown format for further analysis.
-- **Batch Processing**: Can handle multiple files at once, improving work efficiency.
-- **Customizable Configuration**: Users can adjust processing parameters according to their needs to meet different business requirements.
-- **Cross-platform Compatibility**: This SDK can run on multiple operating systems, including Windows, MacOS, and Linux.
-
-
-## Technology Stack
-
-- **Programming Language**: Python >= 3.10  
-- **Dependency Libraries**:  
-  - PyMuPDF: For PDF file parsing.  
-  - BeautifulSoup: For HTML file parsing.  
-  - python-docx: For DOCX file parsing.  
-  - pandas: For data processing and conversion.  
-  - paddleocr: For parsing scanned PDFs, tables, and images.  
-- **Development Environment**: Visual Studio Code or PyCharm  
-- **Version Control**: Git  
-
-## Usage Instructions
-### Installing the SDK
-- **Installation Commands**:
-  ```bash
-  ## Local Installation
-  python setup.py sdist bdist_wheel
-  pip install dist/datamax-0.1.3-py3-none-any.whl
-  
-  ## Pip Installation
-  pip install pydatamax
-  ```
-  
-
-- **Importing the Code**:
-    ```python
-    # File Parsing
-    from datamax import DataMax
-    
-    ## Handling a Single File in Two Ways
-    # 1. Using a List of Length 1
-    data = DataMax(file_path=[r"docx_files_example/船视宝概述.doc"])
-    data = data.get_data()
-    
-    # 2. Using a String
-    data = DataMax(file_path=r"docx_files_example/船视宝概述.doc")
-    data = data.get_data()
-    
-    ## Handling Multiple Files
-    # 1. Using a List of Length n
-    data = DataMax(file_path=[r"docx_files_example/船视宝概述1.doc", r"docx_files_example/船视宝概述2.doc"])
-    data = data.get_data()
-    
-    # 2. Passing a Folder Path as a String
-    data = DataMax(file_path=r"docx_files_example/")
-    data = data.get_data()
-    
-    # Data Cleaning
-    """
-    Cleaning rules can be found in datamax/utils/data_cleaner.py
-    abnormal: Abnormal cleaning
-    private: Privacy processing
-    filter: Text filtering
-    """
-    # Direct Use: Clean the text parameter directly and return a string
-    dm = DataMax()
-    data = dm.clean_data(method_list=["abnormal", "private"], text="<div></div>你好 18717777777 \n\n\n\n")
-    
-    # Process Use: Use after get_data() to return the complete data structure
-    dm = DataMax(file_path=r"C:\Users\cykro\Desktop\数据库开发手册.pdf", use_ocr=True)
-    data2 = dm.get_data()
-    cleaned_data = dm.clean_data(method_list=["abnormal", "filter", "private"])
-    
-    # Large Model Pre-annotation Supporting any model that can be called via OpenAI SDK
-    data = DataMax(file_path=r"path\to\xxx.docx")
-    parsed_data = data.get_data()
-    # If no custom messages are passed, the default messages in the SDK will be used
-    messages = [
-        {'role': 'system', 'content': 'You are a helpful assistant.'},
-        {'role': 'user', 'content': 'Who are you?'}
-    ]
-    qa_datas = data.get_pre_label(
-        api_key="sk-xxx",
-        base_url="https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions",
-        model_name="qwen-max",
-        chunk_size=500,
-        chunk_overlap=100,
-        question_number=5,
-        max_workers=5,
-        # message=[]
-    )
-    print(f'Annotated result:{qa_datas}')
-    ```
-
-
-## Examples
-    ```python
-    ## docx | doc | epub | html | txt | ppt | pptx | xls | xlsx
-    from datamax import DataMax
-    data = DataMax(file_path=r"docx_files_example/船视宝概述.doc", to_markdown=True)
-    """
-    Parameters: 
-    file_path: Relative file path / Absolute file path
-    to_markdown: Whether to convert to markdown (default value False, directly returns text) This parameter only supports word files (doc | docx)
-    """
-    
-    ## jpg | jpeg | png | ...(image types)
-    data = DataMax(file_path=r"image.jpg", use_mineru=True)
-    """
-    Parameters:
-    file_path: Relative file path / Absolute file path
-    use_mineru: Whether to use MinerU enhancement
-    """
-    
-    ## pdf
-    from datamax import DataMax
-    data = DataMax(file_path=r"docx_files_example/船视宝概述.pdf", use_mineru=True)
-    """
-    Parameters: 
-    file_path: Relative file path / Absolute file path
-    use_mineru: Whether to use MinerU enhancement
-    """
-    ```
-
-## Contribution Guide
-We welcome any form of contribution, whether it is reporting bugs, suggesting new features, or submitting code improvements. Please read our Contributor's Guide to learn how to get started.
-## License
-This project is licensed under the MIT License. For more details, see the LICENSE file.
-
-## Contact Information
-If you encounter any issues during use, or have any suggestions or feedback, please contact us through the following means:
-- Email: cy.kron@foxmail.com | zhibaohe@hotmail.com
-- Project Homepage: GitHub Project Link
-