pydatamax 0.1.13__tar.gz → 0.1.14__tar.gz

pydatamax-0.1.14/PKG-INFO

@@ -0,0 +1,228 @@
+Metadata-Version: 2.4
+Name: pydatamax
+Version: 0.1.14
+Summary: A library for parsing and converting various file formats.
+Home-page: https://github.com/Hi-Dolphin/datamax
+Author: ccy
+Author-email: cy.kron@foxmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: oss2<3.0.0,>=2.19.1
+Requires-Dist: aliyun-python-sdk-core<3.0.0,>=2.16.0
+Requires-Dist: aliyun-python-sdk-kms<3.0.0,>=2.16.5
+Requires-Dist: crcmod<2.0.0,>=1.7
+Requires-Dist: langdetect<2.0.0,>=1.0.9
+Requires-Dist: loguru<1.0.0,>=0.7.3
+Requires-Dist: python-docx<2.0.0,>=1.1.2
+Requires-Dist: python-dotenv<2.0.0,>=1.1.0
+Requires-Dist: pymupdf<2.0.0,>=1.26.0
+Requires-Dist: pypdf<6.0.0,>=5.5.0
+Requires-Dist: openpyxl<4.0.0,>=3.1.5
+Requires-Dist: pandas<3.0.0,>=2.2.3
+Requires-Dist: numpy<3.0.0,>=2.2.6
+Requires-Dist: requests<3.0.0,>=2.32.3
+Requires-Dist: tqdm<5.0.0,>=4.67.1
+Requires-Dist: pydantic<3.0.0,>=2.11.5
+Requires-Dist: pydantic-settings<3.0.0,>=2.9.1
+Requires-Dist: python-magic<1.0.0,>=0.4.27
+Requires-Dist: PyYAML<7.0.0,>=6.0.2
+Requires-Dist: Pillow<12.0.0,>=11.2.1
+Requires-Dist: packaging<25.0,>=24.2
+Requires-Dist: beautifulsoup4<5.0.0,>=4.13.4
+Requires-Dist: minio<8.0.0,>=7.2.15
+Requires-Dist: openai<2.0.0,>=1.82.0
+Requires-Dist: jionlp<2.0.0,>=1.5.23
+Requires-Dist: chardet<6.0.0,>=5.2.0
+Requires-Dist: python-pptx<2.0.0,>=1.0.2
+Requires-Dist: tiktoken<1.0.0,>=0.9.0
+Requires-Dist: markitdown<1.0.0,>=0.1.1
+Requires-Dist: xlrd<3.0.0,>=2.0.1
+Requires-Dist: tabulate<1.0.0,>=0.9.0
+Requires-Dist: unstructured<1.0.0,>=0.17.2
+Requires-Dist: markdown<4.0.0,>=3.8
+Requires-Dist: langchain<1.0.0,>=0.3.0
+Requires-Dist: langchain-community<1.0.0,>=0.3.0
+Requires-Dist: ebooklib==0.19
+Requires-Dist: setuptools
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# DataMax
+
+<div align="center">
+
+[中文](README_zh.md) | **English**
+
+[![PyPI version](https://badge.fury.io/py/pydatamax.svg)](https://badge.fury.io/py/pydatamax) [![Python](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+</div>
+
+A powerful multi-format file parsing, data cleaning, and AI annotation toolkit.
+
+## ✨ Core Features
+
+- 🔄 **Multi-format Support**: PDF, DOCX/DOC, PPT/PPTX, XLS/XLSX, HTML, EPUB, TXT, images, and more
+- 🧹 **Intelligent Cleaning**: Three-layer cleaning process with anomaly detection, privacy protection, and text filtering
+- 🤖 **AI Annotation**: LLM-based automatic data annotation and pre-labeling
+- ⚡ **Batch Processing**: Efficient multi-file parallel processing
+- 🎯 **Easy Integration**: Clean API design, ready to use out of the box
+
+## 🚀 Quick Start
+
+### Installation
+
+```bash
+pip install pydatamax
+```
+
+### Basic Usage
+
+```python
+from datamax import DataMax
+
+# Parse a single file
+dm = DataMax(file_path="document.pdf")
+data = dm.get_data()
+
+# Batch processing
+dm = DataMax(file_path=["file1.docx", "file2.pdf"])
+data = dm.get_data()
+
+# Data cleaning
+cleaned_data = dm.clean_data(method_list=["abnormal", "private", "filter"])
+
+# AI annotation
+qa_data = dm.get_pre_label(
+    api_key="your-api-key",
+    base_url="https://api.openai.com/v1",
+    model_name="gpt-3.5-turbo"
+)
+```
+
+## 📖 Detailed Documentation
+
+### File Parsing
+
+#### Supported Formats
+
+| Format | Extensions | Special Features |
+|--------|------------|------------------|
+| Documents | `.pdf`, `.docx`, `.doc` | OCR support, Markdown conversion |
+| Spreadsheets | `.xlsx`, `.xls` | Structured data extraction |
+| Presentations | `.pptx`, `.ppt` | Slide content extraction |
+| Web | `.html`, `.epub` | Tag parsing |
+| Images | `.jpg`, `.png`, `.jpeg` | OCR text recognition |
+| Text | `.txt` | Automatic encoding detection |
+
+#### Advanced Features
+
+```python
+# Advanced PDF parsing (requires MinerU)
+dm = DataMax(file_path="complex.pdf", use_mineru=True)
+
+# Word to Markdown conversion
+dm = DataMax(file_path="document.docx", to_markdown=True)
+
+# Image OCR
+dm = DataMax(file_path="image.jpg", use_ocr=True)
+```
+
+### Data Cleaning
+
+```python
+# Three cleaning modes
+dm.clean_data(method_list=[
+    "abnormal",  # Anomaly data processing
+    "private",   # Privacy information masking
+    "filter"     # Text filtering and normalization
+])
+```
+
+### AI Annotation
+
+```python
+# Custom annotation tasks
+qa_data = dm.get_pre_label(
+    api_key="sk-xxx",
+    base_url="https://api.provider.com/v1",
+    model_name="model-name",
+    chunk_size=500,        # Text chunk size
+    chunk_overlap=100,     # Overlap length
+    question_number=5,     # Questions per chunk
+    max_workers=5          # Concurrency
+)
+```
+
+## ⚙️ Environment Setup
+
+### Optional Dependencies
+
+#### LibreOffice (DOC file support)
+
+**Ubuntu/Debian:**
+```bash
+sudo apt-get install libreoffice
+```
+
+**Windows:**
+1. Download and install [LibreOffice](https://www.libreoffice.org/download/)
+2. Add to environment variables: `C:\Program Files\LibreOffice\program`
+
+#### MinerU (Advanced PDF parsing)
+
+```bash
+# Create virtual environment
+conda create -n mineru python=3.10
+conda activate mineru
+
+# Install MinerU
+pip install -U "magic-pdf[full]" --extra-index-url https://wheels.myhloli.com
+```
+
+For detailed configuration, please refer to [MinerU Documentation](https://github.com/opendatalab/MinerU)
+
+## 🛠️ Development
+
+### Local Installation
+
+```bash
+git clone https://github.com/Hi-Dolphin/datamax.git
+cd datamax
+pip install -r requirements.txt
+python setup.py install
+```
+
+## 📋 System Requirements
+
+- Python >= 3.10
+- Supports Windows, macOS, Linux
+
+## 🤝 Contributing
+
+Issues and Pull Requests are welcome!
+
+## 📄 License
+
+This project is licensed under the [MIT License](LICENSE).
+
+## 📞 Contact Us
+
+- 📧 Email: cy.kron@foxmail.com
+- 🐛 Issues: [GitHub Issues](https://github.com/Hi-Dolphin/datamax/issues)
+- 📚 Documentation: [Project Homepage](https://github.com/Hi-Dolphin/datamax)
+
+---
+
+⭐ If this project helps you, please give us a star! 

pydatamax-0.1.14/README.md

@@ -0,0 +1,167 @@
+# DataMax
+
+<div align="center">
+
+[中文](README_zh.md) | **English**
+
+[![PyPI version](https://badge.fury.io/py/pydatamax.svg)](https://badge.fury.io/py/pydatamax) [![Python](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+</div>
+
+A powerful multi-format file parsing, data cleaning, and AI annotation toolkit.
+
+## ✨ Core Features
+
+- 🔄 **Multi-format Support**: PDF, DOCX/DOC, PPT/PPTX, XLS/XLSX, HTML, EPUB, TXT, images, and more
+- 🧹 **Intelligent Cleaning**: Three-layer cleaning process with anomaly detection, privacy protection, and text filtering
+- 🤖 **AI Annotation**: LLM-based automatic data annotation and pre-labeling
+- ⚡ **Batch Processing**: Efficient multi-file parallel processing
+- 🎯 **Easy Integration**: Clean API design, ready to use out of the box
+
+## 🚀 Quick Start
+
+### Installation
+
+```bash
+pip install pydatamax
+```
+
+### Basic Usage
+
+```python
+from datamax import DataMax
+
+# Parse a single file
+dm = DataMax(file_path="document.pdf")
+data = dm.get_data()
+
+# Batch processing
+dm = DataMax(file_path=["file1.docx", "file2.pdf"])
+data = dm.get_data()
+
+# Data cleaning
+cleaned_data = dm.clean_data(method_list=["abnormal", "private", "filter"])
+
+# AI annotation
+qa_data = dm.get_pre_label(
+    api_key="your-api-key",
+    base_url="https://api.openai.com/v1",
+    model_name="gpt-3.5-turbo"
+)
+```
+
+## 📖 Detailed Documentation
+
+### File Parsing
+
+#### Supported Formats
+
+| Format | Extensions | Special Features |
+|--------|------------|------------------|
+| Documents | `.pdf`, `.docx`, `.doc` | OCR support, Markdown conversion |
+| Spreadsheets | `.xlsx`, `.xls` | Structured data extraction |
+| Presentations | `.pptx`, `.ppt` | Slide content extraction |
+| Web | `.html`, `.epub` | Tag parsing |
+| Images | `.jpg`, `.png`, `.jpeg` | OCR text recognition |
+| Text | `.txt` | Automatic encoding detection |
+
+#### Advanced Features
+
+```python
+# Advanced PDF parsing (requires MinerU)
+dm = DataMax(file_path="complex.pdf", use_mineru=True)
+
+# Word to Markdown conversion
+dm = DataMax(file_path="document.docx", to_markdown=True)
+
+# Image OCR
+dm = DataMax(file_path="image.jpg", use_ocr=True)
+```
+
+### Data Cleaning
+
+```python
+# Three cleaning modes
+dm.clean_data(method_list=[
+    "abnormal",  # Anomaly data processing
+    "private",   # Privacy information masking
+    "filter"     # Text filtering and normalization
+])
+```
+
+### AI Annotation
+
+```python
+# Custom annotation tasks
+qa_data = dm.get_pre_label(
+    api_key="sk-xxx",
+    base_url="https://api.provider.com/v1",
+    model_name="model-name",
+    chunk_size=500,        # Text chunk size
+    chunk_overlap=100,     # Overlap length
+    question_number=5,     # Questions per chunk
+    max_workers=5          # Concurrency
+)
+```
+
+## ⚙️ Environment Setup
+
+### Optional Dependencies
+
+#### LibreOffice (DOC file support)
+
+**Ubuntu/Debian:**
+```bash
+sudo apt-get install libreoffice
+```
+
+**Windows:**
+1. Download and install [LibreOffice](https://www.libreoffice.org/download/)
+2. Add to environment variables: `C:\Program Files\LibreOffice\program`
+
+#### MinerU (Advanced PDF parsing)
+
+```bash
+# Create virtual environment
+conda create -n mineru python=3.10
+conda activate mineru
+
+# Install MinerU
+pip install -U "magic-pdf[full]" --extra-index-url https://wheels.myhloli.com
+```
+
+For detailed configuration, please refer to [MinerU Documentation](https://github.com/opendatalab/MinerU)
+
+## 🛠️ Development
+
+### Local Installation
+
+```bash
+git clone https://github.com/Hi-Dolphin/datamax.git
+cd datamax
+pip install -r requirements.txt
+python setup.py install
+```
+
+## 📋 System Requirements
+
+- Python >= 3.10
+- Supports Windows, macOS, Linux
+
+## 🤝 Contributing
+
+Issues and Pull Requests are welcome!
+
+## 📄 License
+
+This project is licensed under the [MIT License](LICENSE).
+
+## 📞 Contact Us
+
+- 📧 Email: cy.kron@foxmail.com
+- 🐛 Issues: [GitHub Issues](https://github.com/Hi-Dolphin/datamax/issues)
+- 📚 Documentation: [Project Homepage](https://github.com/Hi-Dolphin/datamax)
+
+---
+
+⭐ If this project helps you, please give us a star! 

{pydatamax-0.1.13 → pydatamax-0.1.14}/datamax/loader/core.py

@@ -1,7 +1,7 @@
 import os
 from typing import List
-from datamax.loader.MinioHandler import MinIOClient
-from datamax.loader.OssHandler import OssClient
+from datamax.loader.minio_handler import MinIOClient
+from datamax.loader.oss_handler import OssClient
 
 
 class DataLoader:

{pydatamax-0.1.13 → pydatamax-0.1.14}/datamax/parser/doc_parser.py

@@ -1,14 +1,14 @@
+import logging
 import os
 import shutil
 import subprocess
 import tempfile
-import chardet
-import logging
 from pathlib import Path
 from typing import Union
-from datamax.parser.base import BaseLife
-from datamax.parser.base import MarkdownOutputVo
 
+import chardet
+
+from datamax.parser.base import BaseLife, MarkdownOutputVo
 
 # 配置日志
 logger = logging.getLogger(__name__)
@@ -24,37 +24,41 @@ class DocParser(BaseLife):
     def doc_to_txt(self, doc_path: str, dir_path: str) -> str:
         """将.doc文件转换为.txt文件"""
         logger.info(f"🔄 开始转换DOC文件为TXT - 源文件: {doc_path}, 输出目录: {dir_path}")
-        
+
         try:
             cmd = f'soffice --headless --convert-to txt "{doc_path}" --outdir "{dir_path}"'
             logger.debug(f"⚡ 执行转换命令: {cmd}")
-            
-            process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+            process = subprocess.Popen(
+                cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            )
             stdout, stderr = process.communicate()
             exit_code = process.returncode
-            
+
             if exit_code == 0:
                 logger.info(f"✅ DOC到TXT转换成功 - 退出码: {exit_code}")
                 if stdout:
                     logger.debug(f"📄 转换输出: {stdout.decode('utf-8', errors='replace')}")
             else:
-                encoding = chardet.detect(stderr)['encoding']
+                encoding = chardet.detect(stderr)["encoding"]
                 if encoding is None:
-                    encoding = 'utf-8'
-                error_msg = stderr.decode(encoding, errors='replace')
+                    encoding = "utf-8"
+                error_msg = stderr.decode(encoding, errors="replace")
                 logger.error(f"❌ DOC到TXT转换失败 - 退出码: {exit_code}, 错误信息: {error_msg}")
-                raise Exception(f"Error Output (detected encoding: {encoding}): {error_msg}")
-            
+                raise Exception(
+                    f"Error Output (detected encoding: {encoding}): {error_msg}"
+                )
+
             fname = str(Path(doc_path).stem)
-            txt_path = os.path.join(dir_path, f'{fname}.txt')
-            
+            txt_path = os.path.join(dir_path, f"{fname}.txt")
+
             if not os.path.exists(txt_path):
                 logger.error(f"❌ 转换后的TXT文件不存在: {txt_path}")
                 raise Exception(f"文件转换失败 {doc_path} ==> {txt_path}")
             else:
                 logger.info(f"🎉 TXT文件转换成功，文件路径: {txt_path}")
                 return txt_path
-                
+
         except subprocess.SubprocessError as e:
             logger.error(f"💥 subprocess执行失败: {str(e)}")
             raise Exception(f"执行转换命令时发生错误: {str(e)}")
@@ -65,25 +69,25 @@ class DocParser(BaseLife):
     def read_txt_file(self, txt_path: str) -> str:
         """读取txt文件内容"""
         logger.info(f"📖 开始读取TXT文件: {txt_path}")
-        
+
         try:
             # 检测文件编码
-            with open(txt_path, 'rb') as f:
+            with open(txt_path, "rb") as f:
                 raw_data = f.read()
-                encoding = chardet.detect(raw_data)['encoding']
+                encoding = chardet.detect(raw_data)["encoding"]
                 if encoding is None:
-                    encoding = 'utf-8'
+                    encoding = "utf-8"
                 logger.debug(f"🔍 检测到文件编码: {encoding}")
-            
+
             # 读取文件内容
-            with open(txt_path, 'r', encoding=encoding, errors='replace') as f:
+            with open(txt_path, "r", encoding=encoding, errors="replace") as f:
                 content = f.read()
-            
+
             logger.info(f"📄 TXT文件读取完成 - 内容长度: {len(content)} 字符")
             logger.debug(f"👀 前100字符预览: {content[:100]}...")
-            
+
             return content
-            
+
         except FileNotFoundError as e:
             logger.error(f"🚫 TXT文件未找到: {str(e)}")
             raise Exception(f"文件未找到: {txt_path}")
@@ -94,27 +98,27 @@ class DocParser(BaseLife):
     def read_doc_file(self, doc_path: str) -> str:
         """读取doc文件并转换为文本"""
         logger.info(f"📖 开始读取DOC文件 - 文件: {doc_path}")
-        
+
         try:
             with tempfile.TemporaryDirectory() as temp_path:
                 logger.debug(f"📁 创建临时目录: {temp_path}")
-                
+
                 temp_dir = Path(temp_path)
-                
+
                 file_path = temp_dir / "tmp.doc"
                 shutil.copy(doc_path, file_path)
                 logger.debug(f"📋 复制文件到临时目录: {doc_path} -> {file_path}")
-                
+
                 # 转换DOC为TXT
                 txt_file_path = self.doc_to_txt(str(file_path), str(temp_path))
                 logger.info(f"🎯 DOC转TXT完成: {txt_file_path}")
-                
+
                 # 读取TXT文件内容
                 content = self.read_txt_file(txt_file_path)
                 logger.info(f"✨ TXT文件内容读取完成，内容长度: {len(content)} 字符")
-                
+
                 return content
-                    
+
         except FileNotFoundError as e:
             logger.error(f"🚫 文件未找到: {str(e)}")
             raise Exception(f"文件未找到: {doc_path}")
@@ -128,24 +132,24 @@ class DocParser(BaseLife):
     def parse(self, file_path: str):
         """解析DOC文件"""
         logger.info(f"🎬 开始解析DOC文件: {file_path}")
-        
+
         try:
             # 验证文件存在
             if not os.path.exists(file_path):
                 logger.error(f"🚫 文件不存在: {file_path}")
                 raise FileNotFoundError(f"文件不存在: {file_path}")
-            
+
             # 验证文件大小
             file_size = os.path.getsize(file_path)
             logger.info(f"📏 文件大小: {file_size} 字节")
-            
+
             title = self.get_file_extension(file_path)
             logger.debug(f"🏷️ 提取文件标题: {title}")
-            
+
             # 使用soffice转换为txt后读取内容
             logger.info("📝 使用soffice转换DOC为TXT并读取内容")
             content = self.read_doc_file(doc_path=file_path)
-            
+
             # 根据to_markdown参数决定是否保持原格式还是处理为markdown格式
             if self.to_markdown:
                 # 简单的文本到markdown转换（保持段落结构）
@@ -154,22 +158,26 @@ class DocParser(BaseLife):
             else:
                 mk_content = content
                 logger.info("📝 保持原始文本格式")
-            
+
             logger.info(f"🎊 文件内容解析完成，最终内容长度: {len(mk_content)} 字符")
-            
-            lifecycle = self.generate_lifecycle(source_file=file_path, domain="Technology",
-                                                usage_purpose="Documentation", life_type="LLM_ORIGIN")
+
+            lifecycle = self.generate_lifecycle(
+                source_file=file_path,
+                domain="Technology",
+                usage_purpose="Documentation",
+                life_type="LLM_ORIGIN",
+            )
             logger.debug("⚙️ 生成lifecycle信息完成")
-            
+
             output_vo = MarkdownOutputVo(title, mk_content)
             output_vo.add_lifecycle(lifecycle)
-            
+
             result = output_vo.to_dict()
             logger.info(f"🏆 DOC文件解析完成: {file_path}")
             logger.debug(f"🔑 返回结果键: {list(result.keys())}")
-            
+
             return result
-            
+
         except Exception as e:
             logger.error(f"💀 解析DOC文件失败: {file_path}, 错误: {str(e)}")
             raise
@@ -178,18 +186,18 @@ class DocParser(BaseLife):
         """将纯文本格式化为简单的markdown格式"""
         if not content.strip():
             return content
-        
-        lines = content.split('\n')
+
+        lines = content.split("\n")
         formatted_lines = []
-        
+
         for line in lines:
             line = line.strip()
             if not line:
-                formatted_lines.append('')
+                formatted_lines.append("")
                 continue
-            
+
             # 简单的markdown格式化规则
             # 可以根据需要扩展更多规则
             formatted_lines.append(line)
-        
-        return '\n'.join(formatted_lines)
+
+        return "\n".join(formatted_lines)