xparse-client 0.2.19__py3-none-any.whl → 0.3.0b8__py3-none-any.whl

xparse_client/_http.py

@@ -0,0 +1,351 @@
+"""HTTP 客户端抽象层
+
+基于 httpx 的 HTTP 客户端封装。
+
+Features:
+    - 自动重试（指数退避）
+    - 统一的错误处理
+    - 请求日志
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any
+
+import httpx
+
+from ._config import RetryConfiguration, SDKConfiguration
+from .exceptions import (
+    APIError,
+    AuthenticationError,
+    NotFoundError,
+    PermissionDeniedError,
+    RateLimitError,
+    RequestTimeoutError,
+    ServerError,
+    ValidationError,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _extract_error_message(response: httpx.Response) -> str:
+    """从响应中提取错误信息
+
+    支持多种响应格式:
+    - JSON: {"message": "..."} 或 {"error": "..."} 或 {"detail": "..."}
+    - 纯文本
+
+    Args:
+        response: HTTP 响应
+
+    Returns:
+        错误信息字符串
+    """
+    try:
+        data = response.json()
+        if isinstance(data, dict):
+            # 尝试多种常见的错误字段
+            for key in ("message", "error", "detail", "msg", "error_message"):
+                if key in data:
+                    return str(data[key])
+            # 如果有嵌套的 error 对象
+            if "error" in data and isinstance(data["error"], dict):
+                error_dict = data["error"]
+                return str(error_dict.get("message", error_dict))
+        return str(data)
+    except Exception:
+        # 非 JSON 响应，返回文本内容
+        text = response.text.strip()
+        if text:
+            return text[:500]  # 截断过长的响应
+        return f"HTTP {response.status_code}"
+
+
+def _extract_request_id(response: httpx.Response) -> str | None:
+    """从响应头中提取 request_id
+
+    Args:
+        response: HTTP 响应
+
+    Returns:
+        request_id 或 None
+    """
+    # 尝试多种常见的 request id 头
+    for header in ("x-request-id", "x-req-id", "request-id", "X-Request-Id"):
+        if header in response.headers:
+            return response.headers[header]
+    return None
+
+
+def raise_for_status(response: httpx.Response) -> None:
+    """根据状态码和业务 code 抛出相应异常
+
+    Args:
+        response: HTTP 响应
+
+    Raises:
+        ValidationError: 400 请求参数错误（HTTP 400 或业务 code 400）
+        AuthenticationError: 401 认证失败
+        PermissionDeniedError: 403 权限不足
+        NotFoundError: 404 资源不存在
+        RateLimitError: 429 请求限流
+        ServerError: 5xx 服务器错误
+        APIError: 其他错误
+
+    Note:
+        会检查两层错误：
+        1. HTTP 状态码（非 2xx）
+        2. 业务 code（HTTP 200 但 code 非 200）
+    """
+    # 先检查 HTTP 状态码
+    if not response.is_success:
+        status_code = response.status_code
+        request_id = _extract_request_id(response)
+        message = _extract_error_message(response)
+        response_body = response.text[:1000] if response.text else None
+
+        common_kwargs: dict[str, Any] = {
+            "status_code": status_code,
+            "request_id": request_id,
+            "response_body": response_body,
+        }
+
+        if status_code == 400:
+            raise ValidationError(message, details=common_kwargs)
+        elif status_code == 401:
+            raise AuthenticationError(message, **common_kwargs)
+        elif status_code == 403:
+            raise PermissionDeniedError(message, **common_kwargs)
+        elif status_code == 404:
+            raise NotFoundError(message, **common_kwargs)
+        elif status_code == 429:
+            retry_after = response.headers.get("Retry-After")
+            retry_after_int = int(retry_after) if retry_after and retry_after.isdigit() else None
+            raise RateLimitError(message, retry_after=retry_after_int, **common_kwargs)
+        elif status_code >= 500:
+            raise ServerError(message, **common_kwargs)
+        else:
+            raise APIError(message, **common_kwargs)
+
+    # 检查业务 code（HTTP 200 但业务失败）
+    try:
+        data = response.json()
+        if isinstance(data, dict):
+            code = data.get("code")
+            if code is not None and code != 200:
+                # 提取错误信息
+                request_id = data.get("x_request_id") or _extract_request_id(response)
+                message = data.get("message", f"业务错误 code: {code}")
+                response_body = response.text[:1000] if response.text else None
+
+                error_kwargs: dict[str, Any] = {
+                    "request_id": request_id,
+                    "response_body": response_body,
+                }
+
+                # 根据业务 code 映射到相应异常
+                if code == 400:
+                    raise ValidationError(message, details=error_kwargs)
+                elif code == 401:
+                    raise AuthenticationError(message, **error_kwargs)
+                elif code == 403:
+                    raise PermissionDeniedError(message, **error_kwargs)
+                elif code == 404:
+                    raise NotFoundError(message, **error_kwargs)
+                elif code >= 500:
+                    raise ServerError(message, **error_kwargs)
+                else:
+                    raise APIError(message, **error_kwargs)
+    except (ValueError, KeyError):
+        # 如果无法解析 JSON 或没有 code 字段，认为是成功
+        pass
+
+
+class HTTPClient:
+    """HTTP 客户端
+
+    基于 httpx 的 HTTP 客户端，提供自动重试和错误处理。
+
+    Attributes:
+        config: SDK 配置
+        retry_config: 重试配置
+        _client: httpx.Client 实例
+
+    Example:
+        >>> config = SDKConfiguration(app_id="xxx", secret_code="xxx")
+        >>> http = HTTPClient(config)
+        >>> response = http.request("GET", "/api/xparse/parse")
+    """
+
+    def __init__(
+        self,
+        config: SDKConfiguration,
+        retry_config: RetryConfiguration | None = None,
+        client: httpx.Client | None = None,
+    ) -> None:
+        """初始化 HTTP 客户端
+
+        Args:
+            config: SDK 配置
+            retry_config: 重试配置（可选）
+            client: 自定义 httpx.Client（可选，用于测试）
+        """
+        self.config = config
+        self.retry_config = retry_config or RetryConfiguration(
+            max_retries=config.max_retries,
+            backoff_base=config.backoff_base,
+            backoff_max=config.backoff_max,
+        )
+
+        # 使用传入的 client 或创建新的
+        self._client = client or httpx.Client(
+            base_url=config.get_base_url(),
+            headers=config.get_auth_headers(),
+            timeout=httpx.Timeout(config.timeout),
+        )
+        self._owns_client = client is None
+
+    def request(
+        self,
+        method: str,
+        path: str,
+        *,
+        json: dict[str, Any] | None = None,
+        data: dict[str, Any] | None = None,
+        files: dict[str, Any] | None = None,
+        params: dict[str, Any] | None = None,
+        headers: dict[str, str] | None = None,
+        timeout: float | None = None,
+    ) -> httpx.Response:
+        """发送 HTTP 请求
+
+        支持自动重试和错误处理。
+
+        Args:
+            method: HTTP 方法
+            path: 请求路径
+            json: JSON 请求体
+            data: 表单数据
+            files: 文件数据
+            params: 查询参数
+            headers: 额外的请求头
+            timeout: 请求超时（覆盖默认值）
+
+        Returns:
+            httpx.Response
+
+        Raises:
+            APIError: API 调用失败
+            RequestTimeoutError: 请求超时
+        """
+        request_kwargs: dict[str, Any] = {
+            "method": method,
+            "url": path,
+        }
+
+        if json is not None:
+            request_kwargs["json"] = json
+        if data is not None:
+            request_kwargs["data"] = data
+        if files is not None:
+            request_kwargs["files"] = files
+        if params is not None:
+            request_kwargs["params"] = params
+        if headers is not None:
+            request_kwargs["headers"] = headers
+        if timeout is not None:
+            request_kwargs["timeout"] = timeout
+
+        last_exception: Exception | None = None
+
+        for attempt in range(self.retry_config.max_retries + 1):
+            try:
+                logger.debug(
+                    f"HTTP {method} {path} (attempt {attempt + 1}/{self.retry_config.max_retries + 1})"
+                )
+
+                response = self._client.request(**request_kwargs)
+
+                # 检查是否需要重试
+                if self.retry_config.should_retry(response.status_code, attempt):
+                    backoff = self.retry_config.calculate_backoff(attempt)
+                    logger.warning(
+                        f"请求失败 (status={response.status_code})，{backoff:.1f}s 后重试"
+                    )
+                    time.sleep(backoff)
+                    continue
+
+                # 抛出错误或返回响应
+                raise_for_status(response)
+                return response
+
+            except httpx.TimeoutException as e:
+                last_exception = e
+                if attempt < self.retry_config.max_retries:
+                    backoff = self.retry_config.calculate_backoff(attempt)
+                    logger.warning(f"请求超时，{backoff:.1f}s 后重试")
+                    time.sleep(backoff)
+                    continue
+                raise RequestTimeoutError(
+                    f"请求超时: {path}",
+                    timeout_seconds=timeout or self.config.timeout,
+                ) from e
+
+            except httpx.RequestError as e:
+                last_exception = e
+                if attempt < self.retry_config.max_retries:
+                    backoff = self.retry_config.calculate_backoff(attempt)
+                    logger.warning(f"请求错误: {e}，{backoff:.1f}s 后重试")
+                    time.sleep(backoff)
+                    continue
+                raise APIError(f"请求失败: {e}") from e
+
+            except (
+                AuthenticationError,
+                PermissionDeniedError,
+                NotFoundError,
+                ValidationError,
+            ):
+                # 这些错误不需要重试
+                raise
+
+        # 不应该到达这里，但为了安全
+        if last_exception:
+            raise APIError(f"请求失败: {last_exception}") from last_exception
+        raise APIError("请求失败: 未知错误")
+
+    def get(self, path: str, **kwargs) -> httpx.Response:
+        """发送 GET 请求"""
+        return self.request("GET", path, **kwargs)
+
+    def post(self, path: str, **kwargs) -> httpx.Response:
+        """发送 POST 请求"""
+        return self.request("POST", path, **kwargs)
+
+    def put(self, path: str, **kwargs) -> httpx.Response:
+        """发送 PUT 请求"""
+        return self.request("PUT", path, **kwargs)
+
+    def delete(self, path: str, **kwargs) -> httpx.Response:
+        """发送 DELETE 请求"""
+        return self.request("DELETE", path, **kwargs)
+
+    def close(self) -> None:
+        """关闭客户端"""
+        if self._owns_client:
+            self._client.close()
+
+    def __enter__(self) -> HTTPClient:
+        return self
+
+    def __exit__(self, *args) -> None:
+        self.close()
+
+
+__all__ = [
+    "HTTPClient",
+    "raise_for_status",
+]

xparse_client/api/__init__.py

@@ -0,0 +1,14 @@
+"""API 模块
+
+提供各种 API 的实现类。
+"""
+
+from .extract import Extract
+from .parse import Parse
+from .pipeline import PipelineAPI
+
+__all__ = [
+    "Parse",
+    "Extract",
+    "PipelineAPI",
+]

xparse_client/api/extract.py

@@ -0,0 +1,109 @@
+"""Extract API - 信息抽取
+
+执行 parse + extract Pipeline，用于结构化信息抽取。
+
+Example:
+    >>> result = client.extract.extract(
+    ...     file=file_bytes,
+    ...     filename="invoice.pdf",
+    ...     extract_config=ExtractConfig(schema={"type": "object", ...})
+    ... )
+"""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+from .._base import BaseAPI
+from ..models.extract import ExtractConfig
+from ..models.parse import ParseConfig, ParseResponse
+
+if TYPE_CHECKING:
+    pass
+
+
+class Extract(BaseAPI):
+    """Extract API - 信息抽取
+
+    执行 parse + extract Pipeline，从文档中抽取结构化信息。
+
+    Example:
+        >>> schema = {
+        ...     "type": "object",
+        ...     "properties": {
+        ...         "invoice_number": {"type": "string"},
+        ...         "total_amount": {"type": "number"}
+        ...     }
+        ... }
+        >>> result = client.extract.extract(
+        ...     file=file_bytes,
+        ...     filename="invoice.pdf",
+        ...     extract_config=ExtractConfig(schema=schema)
+        ... )
+    """
+
+    _base_path = "/api/xparse"
+
+    def extract(
+        self,
+        *,
+        file: bytes,
+        filename: str,
+        parse_config: ParseConfig | None = None,
+        extract_config: ExtractConfig | None = None,
+    ) -> ParseResponse:
+        """同步信息抽取
+
+        从文档中抽取结构化信息。
+
+        Args:
+            file: 文件内容
+            filename: 文件名
+            parse_config: 解析配置（可选）
+            extract_config: 抽取配置（必须，包含 schema）
+
+        Returns:
+            ParseResponse: 抽取结果
+
+        Raises:
+            ValueError: 如果未提供 extract_config
+
+        Example:
+            >>> result = client.extract.extract(
+            ...     file=file_bytes,
+            ...     filename="invoice.pdf",
+            ...     extract_config=ExtractConfig(
+            ...         schema={
+            ...             "type": "object",
+            ...             "properties": {
+            ...                 "invoice_number": {"type": "string"},
+            ...                 "total_amount": {"type": "number"}
+            ...             }
+            ...         },
+            ...         generate_citations=True
+            ...     )
+            ... )
+        """
+        files = {"file": (filename, file)}
+        data = {}
+
+        if parse_config:
+            data["parse_config"] = json.dumps(
+                parse_config.model_dump(), ensure_ascii=False
+            )
+
+        # 处理 extract_config
+        if extract_config:
+            data["extract_config"] = json.dumps(
+                extract_config.model_dump(), ensure_ascii=False
+            )
+        else:
+            raise ValueError("extract_config is required")
+
+        response = self._post("/extract/sync", files=files, data=data)
+        return self._parse_response(response, ParseResponse)
+
+
+
+__all__ = ["Extract"]

xparse_client/api/local.py

@@ -0,0 +1,225 @@
+"""Local API - 本地批处理
+
+提供本地同步批处理工作流执行功能。
+
+Example:
+    >>> from xparse_client import XParseClient
+    >>> from xparse_client.connectors import LocalSource, MilvusDestination
+    >>> from xparse_client.models import PipelineStage, ParseConfig
+    >>>
+    >>> client = XParseClient(app_id="...", secret_code="...")
+    >>>
+    >>> result = client.local.run_workflow(
+    ...     source=LocalSource(directory="./docs", pattern=["*.pdf"]),
+    ...     destination=MilvusDestination(
+    ...         db_path="./vectors.db",
+    ...         collection_name="documents",
+    ...         dimension=1024
+    ...     ),
+    ...     stages=[
+    ...         PipelineStage(type="parse", config=ParseConfig(provider="textin")),
+    ...     ]
+    ... )
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Callable, Literal
+
+from .._base import BaseAPI
+from ..models.local import FailedFile, WorkflowResult
+from ..models.pipeline import PipelineConfig, PipelineStage
+
+if TYPE_CHECKING:
+    pass
+
+
+class Local(BaseAPI):
+    """Local API - 本地批处理
+
+    提供本地同步批处理工作流执行，在用户代码中遍历文件并调用 API。
+
+    与 workflows 的区别:
+        - local.run_workflow: 同步阻塞执行，需要本地运行 Python
+        - workflows: 异步非阻塞，在服务端执行，支持 cron 定时
+
+    Attributes:
+        _base_path: API 路径前缀
+    """
+
+    _base_path = "/api/xparse"
+
+    def run_workflow(
+        self,
+        source,  # Union[LocalSource, S3Source, FtpSource, SmbSource]
+        destination,  # Union[LocalDestination, S3Destination, MilvusDestination, QdrantDestination]
+        stages: list[PipelineStage | dict[str, Any]],
+        *,
+        pipeline_config: PipelineConfig | None = None,
+        progress_callback: Callable[[int, int, str], None] | None = None,
+        on_error: Literal["stop", "continue", "retry"] = "stop",
+        max_retries: int = 3,
+    ) -> WorkflowResult:
+        """执行本地同步批处理工作流
+
+        在你的代码中遍历 source 文件，逐个调用 API，写入 destination。
+        这是同步执行的，会阻塞直到所有文件处理完成。
+
+        如果需要:
+        - 异步执行（不阻塞）
+        - 定时任务（cron）
+        - 无需本地常驻
+
+        请使用 client.workflows 在服务端配置工作流。
+
+        Args:
+            source: 数据源（本地 Connector 对象）
+            destination: 输出目的地（本地 Connector 对象）
+            stages: 处理阶段列表
+            pipeline_config: Pipeline 配置（包含中间结果保存配置）
+            progress_callback: 进度回调函数 (current, total, message) -> None
+            on_error: 错误处理策略 ("stop"|"continue"|"retry")
+            max_retries: 最大重试次数（on_error="retry" 时生效）
+
+        Returns:
+            WorkflowResult: 工作流执行结果
+
+        Example:
+            >>> result = client.local.run_workflow(
+            ...     source=LocalSource(directory="./docs"),
+            ...     destination=MilvusDestination(...),
+            ...     stages=[PipelineStage(type="parse", config=ParseConfig())]
+            ... )
+            >>> print(f"成功: {result.success}/{result.total}")
+
+        参考:
+            client.workflows.create() - 创建远程工作流
+            client.workflows.run() - 运行远程工作流
+        """
+        import time
+        from pathlib import Path
+
+        from ..exceptions import APIError
+
+        # 统计信息
+        start_time = time.time()
+        total = 0
+        success = 0
+        failed = 0
+        failed_files = []
+
+        # 列出所有文件
+        files = source.list_files()
+        total = len(files)
+
+        # 处理每个文件
+        for idx, file_path in enumerate(files, start=1):
+            # 进度回调
+            if progress_callback:
+                progress_callback(idx, total, f"处理文件 {file_path}")
+
+            try:
+                # 读取文件
+                file_bytes, data_source = source.read_file(file_path)
+
+                # 获取文件名（只取最后一部分）
+                filename = Path(file_path).name
+
+                # 调用 Pipeline API
+                from .pipeline import PipelineAPI
+
+                pipeline_api = PipelineAPI(self._config, self._http)
+                result = pipeline_api.execute(
+                    file=file_bytes,
+                    filename=filename,
+                    stages=stages,
+                    config=pipeline_config,
+                    data_source=data_source,
+                )
+
+                # 处理中间结果
+                if (
+                    pipeline_config
+                    and pipeline_config.include_intermediate_results
+                    and pipeline_config.intermediate_results_destination
+                    and hasattr(result, "intermediate_results")
+                    and result.intermediate_results
+                ):
+                    for stage_result in result.intermediate_results:
+                        stage_name = stage_result.get("stage")
+                        elements = stage_result.get("elements", [])
+                        if stage_name and elements:
+                            # 转换 elements 为字典列表
+                            elements_data = []
+                            for elem in elements:
+                                if hasattr(elem, "model_dump"):
+                                    elements_data.append(elem.model_dump())
+                                elif isinstance(elem, dict):
+                                    elements_data.append(elem)
+                                else:
+                                    elements_data.append(elem)
+
+                            metadata_with_stage = {
+                                "filename": filename,
+                                "file_path": file_path,
+                                "stage": stage_name,
+                            }
+                            if result.stats and hasattr(result.stats, "record_id"):
+                                metadata_with_stage["record_id"] = result.stats.record_id
+
+                            pipeline_config.intermediate_results_destination.write(
+                                elements_data, metadata_with_stage
+                            )
+
+                # 准备写入数据
+                # 如果是向量数据库，写入 embeddings
+                if hasattr(result, "elements") and result.elements:
+                    elements_data = []
+                    for elem in result.elements:
+                        elem_dict = elem.model_dump() if hasattr(elem, "model_dump") else elem
+                        elements_data.append(elem_dict)
+                else:
+                    elements_data = []
+
+                # 写入目的地
+                metadata = {
+                    "filename": filename,
+                    "file_path": file_path,
+                }
+                if result.stats and hasattr(result.stats, "record_id"):
+                    metadata["record_id"] = result.stats.record_id
+
+                destination.write(elements_data, metadata)
+
+                success += 1
+
+            except (APIError, Exception) as e:
+                error_msg = str(e)
+
+                if on_error == "stop":
+                    # 立即抛出异常，停止处理
+                    raise
+                else:
+                    # continue: 记录失败并继续下一个文件
+                    failed += 1
+                    failed_files.append(
+                        FailedFile(
+                            file_path=file_path,
+                            error=error_msg,
+                            retry_count=0,  # HTTP客户端内部已经处理了重试
+                        )
+                    )
+
+        # 计算总耗时
+        duration = time.time() - start_time
+
+        return WorkflowResult(
+            total=total,
+            success=success,
+            failed=failed,
+            failed_files=failed_files,
+            duration=duration,
+        )
+
+
+__all__ = ["Local"]