maque 0.2.1__py3-none-any.whl

maque/mllm_data_processor_pipeline/__init__.py

@@ -0,0 +1,17 @@
+"""
+多模态大模型训练数据处理pipeline
+
+一个灵活的、模块化的数据处理流水线，支持Web界面交互和断点续传。
+"""
+
+from .core import DataProcessorPipeline, PipelineStep, PipelineConfig
+from .steps import *
+from .web_app import WebApp
+
+__version__ = "0.1.0"
+__all__ = [
+    "DataProcessorPipeline", 
+    "PipelineStep", 
+    "PipelineConfig",
+    "WebApp"
+]

maque/mllm_data_processor_pipeline/core.py

@@ -0,0 +1,341 @@
+"""
+Pipeline核心架构实现
+"""
+
+import json
+import asyncio
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, asdict
+from typing import Dict, List, Any, Optional, Union, Callable
+from pathlib import Path
+import pandas as pd
+from datetime import datetime
+
+from loguru import logger
+from maque.performance import MeasureTime
+
+
+@dataclass
+class PipelineConfig:
+    """Pipeline配置类"""
+
+    # 输入配置
+    input_file: str = ""
+    input_type: str = "csv"  # csv, excel
+    sheet_name: Optional[str] = None
+
+    # 列映射配置
+    image_columns: List[str] = None  # 图像列名称
+    harmful_image_columns: List[str] = None  # 有害图像列
+    harmful_text_columns: List[str] = None  # 有害文本列
+    text_columns: List[str] = None  # 文本列
+
+    # 输出配置
+    output_dir: str = "./output"
+    checkpoint_dir: str = "./checkpoints"
+
+    # 步骤配置
+    steps_config: Dict[str, Dict[str, Any]] = None
+
+    # MLLM配置
+    mllm_config: Dict[str, Any] = None
+
+    def __post_init__(self):
+        if self.image_columns is None:
+            self.image_columns = []
+        if self.harmful_image_columns is None:
+            self.harmful_image_columns = []
+        if self.harmful_text_columns is None:
+            self.harmful_text_columns = []
+        if self.text_columns is None:
+            self.text_columns = []
+        if self.steps_config is None:
+            self.steps_config = {}
+        if self.mllm_config is None:
+            self.mllm_config = {}
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "PipelineConfig":
+        return cls(**data)
+
+    def save(self, filepath: str):
+        """保存配置到文件"""
+        with open(filepath, "w", encoding="utf-8") as f:
+            json.dump(self.to_dict(), f, ensure_ascii=False, indent=2)
+
+    @classmethod
+    def load(cls, filepath: str) -> "PipelineConfig":
+        """从文件加载配置"""
+        with open(filepath, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        return cls.from_dict(data)
+
+
+@dataclass
+class StepResult:
+    """步骤执行结果"""
+
+    step_name: str
+    success: bool
+    data: pd.DataFrame
+    metadata: Dict[str, Any]
+    error: Optional[str] = None
+    execution_time: Optional[float] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        result = {
+            "step_name": self.step_name,
+            "success": self.success,
+            "metadata": self.metadata,
+            "error": self.error,
+            "execution_time": self.execution_time,
+            "data_shape": self.data.shape if self.data is not None else None,
+            "timestamp": datetime.now().isoformat(),
+        }
+        return result
+
+    def save_checkpoint(self, checkpoint_dir: str):
+        """保存检查点"""
+        checkpoint_path = Path(checkpoint_dir)
+        checkpoint_path.mkdir(parents=True, exist_ok=True)
+
+        # 保存数据
+        data_file = checkpoint_path / f"{self.step_name}_data.csv"
+        if self.data is not None:
+            self.data.to_csv(data_file, index=False, encoding="utf-8")
+
+        # 保存元数据
+        meta_file = checkpoint_path / f"{self.step_name}_metadata.json"
+        with open(meta_file, "w", encoding="utf-8") as f:
+            json.dump(self.to_dict(), f, ensure_ascii=False, indent=2)
+
+    @classmethod
+    def load_checkpoint(
+        cls, step_name: str, checkpoint_dir: str
+    ) -> Optional["StepResult"]:
+        """加载检查点"""
+        checkpoint_path = Path(checkpoint_dir)
+        data_file = checkpoint_path / f"{step_name}_data.csv"
+        meta_file = checkpoint_path / f"{step_name}_metadata.json"
+
+        if not (data_file.exists() and meta_file.exists()):
+            return None
+
+        # 加载数据
+        data = pd.read_csv(data_file, encoding="utf-8")
+
+        # 加载元数据
+        with open(meta_file, "r", encoding="utf-8") as f:
+            metadata_info = json.load(f)
+
+        return cls(
+            step_name=metadata_info["step_name"],
+            success=metadata_info["success"],
+            data=data,
+            metadata=metadata_info["metadata"],
+            error=metadata_info.get("error"),
+            execution_time=metadata_info.get("execution_time"),
+        )
+
+
+class PipelineStep(ABC):
+    """Pipeline步骤基类"""
+
+    def __init__(self, name: str, config: Dict[str, Any] = None):
+        self.name = name
+        self.config = config or {}
+        self.logger = logger
+
+    @abstractmethod
+    async def execute(self, data: pd.DataFrame, config: PipelineConfig) -> StepResult:
+        """执行步骤"""
+        pass
+
+    def validate_input(self, data: pd.DataFrame, config: PipelineConfig) -> bool:
+        """验证输入数据"""
+        return data is not None and not data.empty
+
+    def get_step_config(self, config: PipelineConfig) -> Dict[str, Any]:
+        """获取步骤特定配置"""
+        return config.steps_config.get(self.name, {})
+
+
+class DataProcessorPipeline:
+    """数据处理Pipeline主类"""
+
+    def __init__(self, config: PipelineConfig):
+        self.config = config
+        self.steps: List[PipelineStep] = []
+        self.results: List[StepResult] = []
+        self.logger = logger
+        self.status_callback: Optional[Callable] = None
+
+        # 创建输出目录
+        Path(config.output_dir).mkdir(parents=True, exist_ok=True)
+        Path(config.checkpoint_dir).mkdir(parents=True, exist_ok=True)
+
+    def add_step(self, step: PipelineStep) -> "DataProcessorPipeline":
+        """添加处理步骤"""
+        self.steps.append(step)
+        return self
+
+    def set_status_callback(self, callback: Callable[[str, Dict[str, Any]], None]):
+        """设置状态回调函数"""
+        self.status_callback = callback
+
+    def _notify_status(self, status: str, data: Dict[str, Any] = None):
+        """通知状态更新"""
+        if self.status_callback:
+            self.status_callback(status, data or {})
+
+    async def load_data(self) -> pd.DataFrame:
+        """加载输入数据"""
+        self._notify_status("loading_data", {"file": self.config.input_file})
+
+        if self.config.input_type == "csv":
+            data = pd.read_csv(self.config.input_file, encoding="utf-8")
+        elif self.config.input_type == "excel":
+            # 如果有图像列，使用maque的excel_helper
+            if self.config.image_columns:
+                from maque.utils.excel_helper import extract_excel_with_images
+
+                data = extract_excel_with_images(
+                    excel_path=self.config.input_file,
+                    image_column_names=self.config.image_columns,
+                    sheet_name=self.config.sheet_name,
+                    image_output_dir=Path(self.config.output_dir) / "images",
+                )
+            else:
+                data = pd.read_excel(
+                    self.config.input_file, sheet_name=self.config.sheet_name
+                )
+        else:
+            raise ValueError(f"不支持的输入类型: {self.config.input_type}")
+
+        self.logger.info(f"加载数据完成，共{len(data)}行")
+        return data
+
+    async def run(self, resume_from: Optional[str] = None) -> List[StepResult]:
+        """运行Pipeline"""
+        with MeasureTime("Pipeline执行"):
+            self._notify_status("starting", {"total_steps": len(self.steps)})
+
+            # 加载数据
+            current_data = await self.load_data()
+
+            # 确定开始步骤
+            start_idx = 0
+            if resume_from:
+                # 尝试从检查点恢复
+                checkpoint_result = StepResult.load_checkpoint(
+                    resume_from, self.config.checkpoint_dir
+                )
+                if checkpoint_result:
+                    current_data = checkpoint_result.data
+                    start_idx = (
+                        next(
+                            (
+                                i
+                                for i, step in enumerate(self.steps)
+                                if step.name == resume_from
+                            ),
+                            0,
+                        )
+                        + 1
+                    )
+                    self.results.append(checkpoint_result)
+                    self.logger.info(f"从检查点恢复: {resume_from}")
+
+            # 执行步骤
+            for i, step in enumerate(self.steps[start_idx:], start_idx):
+                self._notify_status(
+                    "executing_step",
+                    {
+                        "step_name": step.name,
+                        "step_index": i,
+                        "total_steps": len(self.steps),
+                    },
+                )
+
+                try:
+                    self.logger.info(f"执行步骤 {i + 1}/{len(self.steps)}: {step.name}")
+
+                    # 验证输入
+                    if not step.validate_input(current_data, self.config):
+                        raise ValueError(f"步骤 {step.name} 输入验证失败")
+
+                    # 执行步骤
+                    result = await step.execute(current_data, self.config)
+
+                    # 保存检查点
+                    result.save_checkpoint(self.config.checkpoint_dir)
+
+                    # 更新当前数据
+                    current_data = result.data
+                    self.results.append(result)
+
+                    self.logger.info(f"步骤 {step.name} 执行完成")
+
+                except Exception as e:
+                    error_msg = f"步骤 {step.name} 执行失败: {str(e)}"
+                    self.logger.error(error_msg)
+
+                    # 创建失败结果
+                    failed_result = StepResult(
+                        step_name=step.name,
+                        success=False,
+                        data=current_data,
+                        metadata={},
+                        error=error_msg,
+                    )
+                    failed_result.save_checkpoint(self.config.checkpoint_dir)
+                    self.results.append(failed_result)
+
+                    self._notify_status(
+                        "step_failed", {"step_name": step.name, "error": error_msg}
+                    )
+
+                    raise
+
+            self._notify_status("completed", {"total_results": len(self.results)})
+            self.logger.info("Pipeline执行完成")
+
+            return self.results
+
+    def get_status(self) -> Dict[str, Any]:
+        """获取Pipeline状态"""
+        return {
+            "total_steps": len(self.steps),
+            "completed_steps": len([r for r in self.results if r.success]),
+            "failed_steps": len([r for r in self.results if not r.success]),
+            "current_step": len(self.results),
+            "steps": [
+                {"name": step.name, "config": step.config} for step in self.steps
+            ],
+            "results": [result.to_dict() for result in self.results],
+        }
+
+    def save_final_results(self):
+        """保存最终结果"""
+        if not self.results:
+            return
+
+        final_result = self.results[-1]
+        if final_result.success and final_result.data is not None:
+            output_file = Path(self.config.output_dir) / "final_output.csv"
+            final_result.data.to_csv(output_file, index=False, encoding="utf-8")
+            self.logger.info(f"最终结果已保存至: {output_file}")
+
+        # 保存执行摘要
+        summary = {
+            "config": self.config.to_dict(),
+            "execution_summary": self.get_status(),
+            "timestamp": datetime.now().isoformat(),
+        }
+
+        summary_file = Path(self.config.output_dir) / "execution_summary.json"
+        with open(summary_file, "w", encoding="utf-8") as f:
+            json.dump(summary, f, ensure_ascii=False, indent=2)

maque/mllm_data_processor_pipeline/example.py

@@ -0,0 +1,291 @@
+#!/usr/bin/env python3
+"""
+MLLM Data Processor Pipeline 使用示例
+
+这个示例展示了如何使用Pipeline进行多模态大模型训练数据处理。
+"""
+
+import asyncio
+import sys
+from pathlib import Path
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from maque.mllm_data_processor_pipeline import (
+    DataProcessorPipeline, 
+    PipelineConfig,
+    WebApp
+)
+from maque.mllm_data_processor_pipeline.steps import *
+
+
+async def example_basic_usage():
+    """基本使用示例"""
+    print("=== 基本使用示例 ===")
+    
+    # 1. 创建配置
+    config = PipelineConfig(
+        input_file="example_data.csv",
+        input_type="csv",
+        image_columns=["image_url"],
+        text_columns=["text", "content"],
+        harmful_image_columns=["harmful_img"],
+        harmful_text_columns=["harmful_text"],
+        output_dir="./output",
+        checkpoint_dir="./checkpoints",
+        mllm_config={
+            "model_name": "gpt-4o",
+            "api_key": "your-api-key-here",
+            "temperature": 0.7,
+            "max_tokens": 2048
+        },
+        steps_config={
+            "data_validation": {
+                "skip_validation": False,
+                "min_text_length": 10,
+                "require_images": False
+            },
+            "mllm_annotation": {
+                "concurrent_config": {
+                    "max_workers": 5,
+                    "rate_limit": 1.0
+                },
+                "annotation_prompts": {
+                    "system_prompt": "你是一个专业的数据标注员，请根据给定的图像和文本内容进行标注。",
+                    "user_prompt_template": """请对以下内容进行标注：
+文本：{text}
+
+请提供：
+1. 内容摘要（50字以内）
+2. 主要标签（用逗号分隔）
+3. 情感倾向（正面/负面/中性）
+4. 质量评估（高/中/低）"""
+                }
+            },
+            "mllm_refinement": {
+                "concurrent_config": {
+                    "max_workers": 3,
+                    "rate_limit": 0.5
+                },
+                "refinement_criteria": {
+                    "quality_threshold": 0.7,
+                    "skip_high_quality": True
+                }
+            },
+            "format_conversion": {
+                "output_formats": ["jsonl", "csv", "json"],
+                "field_mapping": {
+                    "text_field": "text",
+                    "images_field": "images",
+                    "labels_field": "labels"
+                }
+            }
+        }
+    )
+    
+    # 2. 创建Pipeline
+    pipeline = DataProcessorPipeline(config)
+    
+    # 3. 添加处理步骤
+    pipeline.add_step(DataLoaderStep()) \
+           .add_step(DataAlignmentStep()) \
+           .add_step(DataValidationStep()) \
+           .add_step(MllmAnnotationStep()) \
+           .add_step(MllmRefinementStep()) \
+           .add_step(FormatConversionStep()) \
+           .add_step(ResultValidationStep())
+    
+    # 4. 设置状态回调（可选）
+    def status_callback(status, data):
+        print(f"状态更新: {status} - {data}")
+    
+    pipeline.set_status_callback(status_callback)
+    
+    # 5. 运行Pipeline
+    try:
+        results = await pipeline.run()
+        
+        print("\n=== 执行结果 ===")
+        for i, result in enumerate(results):
+            step_name = result.step_name
+            success = "✓" if result.success else "✗"
+            print(f"{i+1}. {step_name}: {success}")
+            if result.error:
+                print(f"   错误: {result.error}")
+        
+        # 6. 保存最终结果
+        pipeline.save_final_results()
+        print("\n处理完成！结果已保存到output目录。")
+        
+    except Exception as e:
+        print(f"Pipeline执行失败: {e}")
+
+
+async def example_resume_from_checkpoint():
+    """从检查点恢复示例"""
+    print("=== 从检查点恢复示例 ===")
+    
+    config = PipelineConfig(
+        input_file="example_data.csv",
+        input_type="csv",
+        checkpoint_dir="./checkpoints"
+    )
+    
+    pipeline = DataProcessorPipeline(config)
+    # 添加步骤...
+    
+    # 从特定步骤恢复
+    try:
+        results = await pipeline.run(resume_from="mllm_annotation")
+        print("从检查点恢复执行成功！")
+    except Exception as e:
+        print(f"从检查点恢复失败: {e}")
+
+
+def example_web_interface():
+    """Web界面示例"""
+    print("=== Web界面示例 ===")
+    
+    # 创建Web应用
+    app = WebApp(static_dir="./static")
+    
+    # 启动Web服务器
+    print("启动Web界面...")
+    print("访问: http://localhost:8000")
+    app.run(host="0.0.0.0", port=8000)
+
+
+def example_custom_step():
+    """自定义步骤示例"""
+    print("=== 自定义步骤示例 ===")
+    
+    class CustomProcessingStep(PipelineStep):
+        """自定义处理步骤"""
+        
+        def __init__(self, name: str = "custom_processing"):
+            super().__init__(name)
+        
+        async def execute(self, data, config):
+            """执行自定义处理"""
+            try:
+                # 自定义处理逻辑
+                processed_data = data.copy()
+                
+                # 添加自定义列
+                processed_data['custom_score'] = 0.85
+                processed_data['custom_tag'] = 'processed'
+                
+                return StepResult(
+                    step_name=self.name,
+                    success=True,
+                    data=processed_data,
+                    metadata={"custom_metric": 42}
+                )
+                
+            except Exception as e:
+                return StepResult(
+                    step_name=self.name,
+                    success=False,
+                    data=data,
+                    metadata={},
+                    error=str(e)
+                )
+    
+    # 使用自定义步骤
+    config = PipelineConfig(input_file="example_data.csv")
+    pipeline = DataProcessorPipeline(config)
+    
+    pipeline.add_step(DataLoaderStep()) \
+           .add_step(CustomProcessingStep()) \
+           .add_step(FormatConversionStep())
+    
+    print("自定义步骤已添加到Pipeline")
+
+
+async def example_with_excel():
+    """Excel文件处理示例"""
+    print("=== Excel文件处理示例 ===")
+    
+    config = PipelineConfig(
+        input_file="example_data.xlsx",
+        input_type="excel",
+        sheet_name=None,  # 使用默认工作表
+        image_columns=["image1", "image2"],  # Excel中包含图像的列
+        text_columns=["title", "description"],
+        output_dir="./excel_output",
+        steps_config={
+            "data_loader": {
+                "extract_images_from_excel": True,
+                "image_output_dir": "extracted_images"
+            }
+        }
+    )
+    
+    pipeline = DataProcessorPipeline(config)
+    pipeline.add_step(DataLoaderStep()) \
+           .add_step(DataAlignmentStep()) \
+           .add_step(FormatConversionStep())
+    
+    try:
+        results = await pipeline.run()
+        print("Excel文件处理完成！")
+    except Exception as e:
+        print(f"Excel文件处理失败: {e}")
+
+
+def create_sample_data():
+    """创建示例数据文件"""
+    import pandas as pd
+    
+    # 创建示例CSV数据
+    sample_data = {
+        'text': [
+            '这是一个关于猫的有趣故事。',
+            '人工智能技术正在快速发展。', 
+            '今天的天气非常好，适合出游。',
+            '这款产品的质量令人担忧。',
+            '学习编程需要持续的练习和思考。'
+        ],
+        'image_url': [
+            'images/cat1.jpg',
+            'images/ai_tech.png',
+            'images/sunny_day.jpg',
+            'images/product.jpg',
+            'images/coding.png'
+        ],
+        'harmful_img': [0, 0, 0, 1, 0],
+        'harmful_text': [0, 0, 0, 1, 0]
+    }
+    
+    df = pd.DataFrame(sample_data)
+    df.to_csv('example_data.csv', index=False, encoding='utf-8')
+    print("示例数据文件 example_data.csv 已创建")
+
+
+def main():
+    """主函数"""
+    import argparse
+    
+    parser = argparse.ArgumentParser(description='MLLM Data Processor Pipeline 示例')
+    parser.add_argument('--mode', choices=['basic', 'resume', 'web', 'custom', 'excel', 'create-sample'], 
+                       default='basic', help='运行模式')
+    
+    args = parser.parse_args()
+    
+    if args.mode == 'create-sample':
+        create_sample_data()
+    elif args.mode == 'basic':
+        asyncio.run(example_basic_usage())
+    elif args.mode == 'resume':
+        asyncio.run(example_resume_from_checkpoint())
+    elif args.mode == 'web':
+        example_web_interface()
+    elif args.mode == 'custom':
+        example_custom_step()
+    elif args.mode == 'excel':
+        asyncio.run(example_with_excel())
+
+
+if __name__ == "__main__":
+    main()

maque/mllm_data_processor_pipeline/steps/__init__.py

@@ -0,0 +1,56 @@
+"""
+Pipeline处理步骤实现
+"""
+
+from .data_loader import DataLoaderStep
+from .data_alignment import DataAlignmentStep
+from .data_validation import DataValidationStep
+from .mllm_annotation import MllmAnnotationStep
+from .mllm_refinement import MllmRefinementStep
+from .format_conversion import FormatConversionStep
+from .result_validation import ResultValidationStep
+
+# 所有可用的步骤类
+ALL_STEPS = [
+    DataLoaderStep,
+    DataAlignmentStep,
+    DataValidationStep,
+    MllmAnnotationStep,
+    MllmRefinementStep,
+    FormatConversionStep,
+    ResultValidationStep
+]
+
+def get_all_steps():
+    """获取所有可用的步骤类"""
+    return ALL_STEPS
+
+def create_step_from_config(step_config: dict):
+    """根据配置创建步骤实例"""
+    step_name = step_config.get("name")
+    step_type = step_config.get("type")
+    step_params = step_config.get("params", {})
+    
+    # 根据类型查找对应的步骤类
+    step_class = None
+    for cls in ALL_STEPS:
+        if cls.__name__ == step_type:
+            step_class = cls
+            break
+    
+    if not step_class:
+        raise ValueError(f"未知的步骤类型: {step_type}")
+    
+    return step_class(name=step_name, config=step_params)
+
+__all__ = [
+    "DataLoaderStep",
+    "DataAlignmentStep", 
+    "DataValidationStep",
+    "MllmAnnotationStep",
+    "MllmRefinementStep",
+    "FormatConversionStep",
+    "ResultValidationStep",
+    "get_all_steps",
+    "create_step_from_config"
+]