xparse-client 0.2.20__py3-none-any.whl → 0.3.0b2__py3-none-any.whl

example/4_advanced_workflow.py

@@ -0,0 +1,327 @@
+"""示例 4: 高级批处理工作流
+
+演示高级批处理功能和最佳实践：
+- 从配置文件加载
+- 自定义 Source/Destination
+- 完整的错误处理
+- 日志记录
+
+运行方式:
+    export TEXTIN_APP_ID="your-app-id"
+    export TEXTIN_SECRET_CODE="your-secret-code"
+    python example/4_advanced_workflow.py
+"""
+
+import json
+import logging
+import os
+from datetime import datetime
+from pathlib import Path
+
+from xparse_client import XParseClient
+from xparse_client.connectors import LocalDestination, LocalSource
+from xparse_client.models import (
+    ChunkConfig,
+    ChunkStage,
+    EmbedConfig,
+    EmbedStage,
+    ParseConfig,
+    ParseStage,
+)
+
+# 配置日志
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
+def example_1_config_file():
+    """示例 4.1: 从配置文件加载"""
+    print("\n" + "="*60)
+    print("示例 4.1: 从配置文件加载配置")
+    print("="*60)
+
+    # 创建配置文件
+    config = {
+        "source": {
+            "directory": "./data/input",
+            "pattern": ["*.pdf", "*.docx", "*.txt"],
+            "recursive": False
+        },
+        "destination": {
+            "output_dir": "./data/output"
+        },
+        "stages": [
+            {
+                "type": "parse",
+                "provider": "textin"
+            },
+            {
+                "type": "chunk",
+                "strategy": "by_title",
+                "max_characters": 1000,
+                "overlap": 50
+            },
+            {
+                "type": "embed",
+                "provider": "qwen",
+                "model_name": "text-embedding-v3"
+            }
+        ],
+        "options": {
+            "on_error": "continue",
+            "max_retries": 3
+        }
+    }
+
+    config_file = Path("workflow_config.json")
+    config_file.write_text(json.dumps(config, indent=2, ensure_ascii=False))
+
+    print(f"\n📄 配置文件已创建: {config_file}")
+    print("\n配置内容:")
+    print(json.dumps(config, indent=2, ensure_ascii=False))
+
+    # 从配置文件加载
+    print("\n⏳ 从配置文件加载...")
+    loaded_config = json.loads(config_file.read_text())
+
+    # 创建客户端
+    XParseClient.from_env()
+
+    # 演示如何从配置构建 source 和 destination（不实际创建）
+    print(f"\n✅ 配置加载成功!")
+    print(f"  - Source 配置:")
+    print(f"      目录: {loaded_config['source']['directory']}")
+    print(f"      模式: {loaded_config['source']['pattern']}")
+    print(f"      递归: {loaded_config['source']['recursive']}")
+    print(f"  - Destination 配置:")
+    print(f"      输出目录: {loaded_config['destination']['output_dir']}")
+
+    # 构建 stages
+    stages = []
+    for stage_config in loaded_config["stages"]:
+        if stage_config["type"] == "parse":
+            stages.append(ParseStage(
+                config=ParseConfig(provider=stage_config["provider"])
+            ))
+        elif stage_config["type"] == "chunk":
+            stages.append(ChunkStage(
+                config=ChunkConfig(
+                    strategy=stage_config["strategy"],
+                    max_characters=stage_config.get("max_characters", 1024),
+                    overlap=stage_config.get("overlap", 0)
+                )
+            ))
+        elif stage_config["type"] == "embed":
+            stages.append(EmbedStage(
+                config=EmbedConfig(
+                    provider=stage_config["provider"],
+                    model_name=stage_config.get("model_name", "text-embedding-v3")
+                )
+            ))
+
+    print(f"  - Stages: {len(stages)} 个")
+    for i, stage in enumerate(stages, 1):
+        print(f"      {i}. {stage.type}")
+
+    # 清理
+    config_file.unlink(missing_ok=True)
+
+
+def example_2_custom_source():
+    """示例 4.2: 自定义 Source（过滤特定文件）"""
+    print("\n" + "="*60)
+    print("示例 4.2: 自定义 Source - 按文件大小过滤")
+    print("="*60)
+
+    from typing import Any
+
+    from xparse_client.connectors.sources import Source
+
+    class FilteredLocalSource(Source):
+        """自定义 Source: 只处理特定大小范围的文件"""
+
+        def __init__(self, directory: str, min_size: int = 0, max_size: int = 10*1024*1024):
+            self.directory = Path(directory)
+            self.min_size = min_size
+            self.max_size = max_size
+
+        def list_files(self) -> list[str]:
+            """列出符合大小要求的文件"""
+            files = []
+            for f in self.directory.glob("**/*.txt"):
+                size = f.stat().st_size
+                if self.min_size <= size <= self.max_size:
+                    files.append(str(f))
+            return files
+
+        def read_file(self, file_path: str) -> tuple[bytes, dict[str, Any]]:
+            """读取文件"""
+            path = Path(file_path)
+            content = path.read_bytes()
+
+            data_source = {
+                "url": f"file://{path.absolute()}",
+                "size": path.stat().st_size,
+                "modified": path.stat().st_mtime,
+            }
+
+            return content, data_source
+
+    # 准备测试
+    test_dir = Path("./test_custom_source")
+    test_dir.mkdir(exist_ok=True)
+
+    # 创建不同大小的文件
+    (test_dir / "small.txt").write_text("小")
+    (test_dir / "medium.txt").write_text("中等" * 100)
+    (test_dir / "large.txt").write_text("大" * 10000)
+
+    try:
+        print(f"\n📁 测试目录: {test_dir}")
+        print(f"  - small.txt: {(test_dir / 'small.txt').stat().st_size} bytes")
+        print(f"  - medium.txt: {(test_dir / 'medium.txt').stat().st_size} bytes")
+        print(f"  - large.txt: {(test_dir / 'large.txt').stat().st_size} bytes")
+
+        # 使用自定义 Source (只处理 100 - 5000 bytes)
+        source = FilteredLocalSource(
+            directory=str(test_dir),
+            min_size=100,
+            max_size=5000
+        )
+
+        files = source.list_files()
+        print(f"\n✅ 过滤结果: {len(files)} 个文件符合条件")
+        for f in files:
+            print(f"  - {Path(f).name}")
+
+    finally:
+        # 清理
+        import shutil
+        if test_dir.exists():
+            shutil.rmtree(test_dir)
+
+
+def example_3_production_workflow():
+    """示例 4.3: 生产环境工作流（完整错误处理和日志）"""
+    print("\n" + "="*60)
+    print("示例 4.3: 生产环境工作流")
+    print("="*60)
+
+    client = XParseClient.from_env()
+
+    # 准备测试
+    source_dir = Path("./test_production")
+    output_dir = Path("./test_production_output")
+    log_dir = Path("./test_production_logs")
+
+    source_dir.mkdir(exist_ok=True)
+    output_dir.mkdir(exist_ok=True)
+    log_dir.mkdir(exist_ok=True)
+
+    # 创建测试文件
+    for i in range(3):
+        (source_dir / f"prod_doc_{i+1}.txt").write_text(f"生产文档 {i+1}\n" * 50)
+
+    # 设置日志文件
+    log_file = log_dir / f"workflow_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log"
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setLevel(logging.INFO)
+    logger.addHandler(file_handler)
+
+    try:
+        logger.info(f"开始工作流: {datetime.now()}")
+        logger.info(f"源目录: {source_dir}")
+        logger.info(f"输出目录: {output_dir}")
+
+        # 进度和错误回调
+
+        def on_progress(current, total, message):
+            logger.info(f"进度: [{current}/{total}] {message}")
+            print(f"  📊 [{current}/{total}] {message}")
+
+        # 定义 stages
+        stages = [
+            ParseStage(config=ParseConfig(provider="textin")),
+            ChunkStage(config=ChunkConfig(strategy="basic")),
+            EmbedStage(config=EmbedConfig(provider="qwen"))
+        ]
+
+        # 执行工作流
+        print(f"\n⏳ 开始处理 (日志: {log_file})...")
+
+        result = client.local.run_workflow(
+            source=LocalSource(directory=str(source_dir)),
+            destination=LocalDestination(output_dir=str(output_dir)),
+            stages=stages,
+            progress_callback=on_progress,
+            on_error="continue",
+            max_retries=2
+        )
+
+        # 记录结果
+        logger.info(f"工作流完成: 成功 {result.success}, 失败 {result.failed}")
+        logger.info(f"耗时: {result.duration:.2f} 秒")
+
+        print("\n✅ 工作流完成!")
+        print(f"  - 总文件: {result.total}")
+        print(f"  - 成功: {result.success}")
+        print(f"  - 失败: {result.failed}")
+        print(f"  - 耗时: {result.duration:.2f} 秒")
+        print(f"  - 日志: {log_file}")
+
+        if result.failed_files:
+            logger.error(f"失败的文件: {len(result.failed_files)} 个")
+            for failed in result.failed_files:
+                logger.error(f"  - {failed.file_path}: {failed.error}")
+
+    except Exception as e:
+        logger.error(f"工作流异常: {e}", exc_info=True)
+        raise
+
+    finally:
+        # 清理
+        import shutil
+        for d in [source_dir, output_dir, log_dir]:
+            if d.exists():
+                shutil.rmtree(d)
+
+
+def main():
+    """主函数"""
+    print("\n" + "="*60)
+    print("xparse-client 高级批处理工作流示例")
+    print("="*60)
+
+    # 检查环境变量
+    if not os.getenv("TEXTIN_APP_ID") or not os.getenv("TEXTIN_SECRET_CODE"):
+        print("\n❌ 错误: 请先设置环境变量:")
+        print("  export TEXTIN_APP_ID='your-app-id'")
+        print("  export TEXTIN_SECRET_CODE='your-secret-code'")
+        return
+
+    print("\n💡 本示例展示:")
+    print("  - 从 JSON 配置文件加载设置")
+    print("  - 自定义 Source 实现（按文件大小过滤）")
+    print("  - 生产环境级别的错误处理和日志")
+
+    try:
+        # 运行示例
+        example_1_config_file()
+        example_2_custom_source()
+        example_3_production_workflow()
+
+        print("\n" + "="*60)
+        print("✅ 所有示例运行完成!")
+        print("="*60)
+
+    except Exception as e:
+        print(f"\n❌ 错误: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    main()

example/README.md

@@ -0,0 +1,128 @@
+# xparse-client 使用示例
+
+本目录包含了 xparse-client SDK 的各种使用示例。
+
+## 📋 示例列表
+
+### 1. 基础 API 使用 - `1_basic_api_usage.py`
+
+演示最基本的 API 调用：
+- 单文件解析（Parse API）
+- 文档提取（Extract API）
+- Pipeline 执行（完整流程）
+
+**适合**: 初次使用、单文件处理
+
+```bash
+python example/1_basic_api_usage.py
+```
+
+---
+
+### 2. 服务端异步任务 - `2_async_job.py`
+
+演示服务端异步任务的使用：
+- 创建异步解析任务
+- 轮询任务状态
+- 等待任务完成
+- 错误处理
+
+**适合**: 处理大文件、长时间任务
+
+```bash
+python example/2_async_job.py
+```
+
+---
+
+### 3. 本地批量处理 - `3_local_workflow.py`
+
+演示本地批量处理工作流：
+- 从本地目录读取文件
+- 批量调用 Pipeline API
+- 写入 Milvus 向量数据库
+- 查看处理统计
+
+**适合**: 批量导入文档到向量数据库
+
+```bash
+python example/3_local_workflow.py
+```
+
+---
+
+### 4. 高级批处理 - `4_advanced_workflow.py`
+
+演示高级批处理功能：
+- 进度回调
+- 错误处理策略（stop/continue/retry）
+- 自定义 Source/Destination
+- 从配置文件加载
+
+**适合**: 生产环境、复杂场景
+
+```bash
+python example/4_advanced_workflow.py
+```
+
+---
+
+## 🔧 环境配置
+
+所有示例都需要配置 TextIn API 凭证。
+
+### 方式 1: 环境变量（推荐）
+
+```bash
+export TEXTIN_APP_ID="your-app-id"
+export TEXTIN_SECRET_CODE="your-secret-code"
+```
+
+### 方式 2: 修改代码
+
+在示例代码中直接修改：
+
+```python
+client = XParseClient(
+    app_id="your-app-id",
+    secret_code="your-secret-code"
+)
+```
+
+## 📝 配置文件示例
+
+查看 `config_example.json` 了解如何从配置文件加载设置。
+
+## 🚀 快速开始
+
+1. 安装依赖
+
+```bash
+pip install xparse-client
+# 或安装所有可选依赖
+pip install xparse-client[all]
+```
+
+2. 配置凭证
+
+```bash
+export TEXTIN_APP_ID="your-app-id"
+export TEXTIN_SECRET_CODE="your-secret-code"
+```
+
+3. 运行示例
+
+```bash
+# 从基础示例开始
+python example/1_basic_api_usage.py
+
+# 然后尝试批量处理
+python example/3_local_workflow.py
+```
+
+## 📚 更多资源
+
+- [完整文档](../README.md)
+- [API 参考](../docs/)
+- [迁移指南](../docs/MIGRATION_FROM_PIPELINE.md)
+- [GitHub Issues](https://github.com/textin/xparse-client/issues)

example/config_example.json

@@ -0,0 +1,95 @@
+{
+  "description": "xparse-client 工作流配置示例",
+  "source": {
+    "type": "local",
+    "directory": "./data/documents",
+    "pattern": ["*.pdf", "*.docx", "*.txt"],
+    "recursive": true,
+    "description": "从本地目录读取文档，支持多种文件类型"
+  },
+  "destination": {
+    "type": "local",
+    "output_dir": "./data/output",
+    "description": "输出到本地 JSON 文件"
+  },
+  "stages": [
+    {
+      "type": "parse",
+      "provider": "textin",
+      "description": "使用 TextIn 解析文档"
+    },
+    {
+      "type": "chunk",
+      "strategy": "by_title",
+      "max_characters": 1000,
+      "overlap": 50,
+      "description": "按标题分块，每块最多 1000 字符，重叠 50 字符"
+    },
+    {
+      "type": "embed",
+      "provider": "qwen",
+      "model_name": "text-embedding-v3",
+      "description": "使用通义千问生成向量"
+    }
+  ],
+  "options": {
+    "on_error": "continue",
+    "max_retries": 3,
+    "description": "遇到错误继续处理，失败重试 3 次"
+  },
+  "milvus_example": {
+    "description": "使用 Milvus 作为目的地的配置示例",
+    "destination": {
+      "type": "milvus",
+      "db_path": "./vectors.db",
+      "collection_name": "documents",
+      "dimension": 1024,
+      "description": "写入本地 Milvus 数据库"
+    }
+  },
+  "s3_example": {
+    "description": "使用 S3 作为数据源的配置示例",
+    "source": {
+      "type": "s3",
+      "endpoint": "https://s3.amazonaws.com",
+      "access_key": "your-access-key",
+      "secret_key": "your-secret-key",
+      "bucket": "your-bucket",
+      "prefix": "documents/",
+      "region": "us-east-1",
+      "pattern": ["*.pdf"],
+      "description": "从 S3 读取 PDF 文件"
+    }
+  },
+  "extract_example": {
+    "description": "使用 Extract 提取结构化数据的配置示例",
+    "stages": [
+      {
+        "type": "parse",
+        "provider": "textin"
+      },
+      {
+        "type": "extract",
+        "schema": {
+          "type": "object",
+          "properties": {
+            "title": {
+              "type": "string",
+              "description": "文档标题"
+            },
+            "author": {
+              "type": "string",
+              "description": "作者名称"
+            },
+            "date": {
+              "type": "string",
+              "description": "日期"
+            }
+          },
+          "required": ["title"]
+        },
+        "description": "提取文档的标题、作者和日期"
+      }
+    ]
+  }
+}