xparse-client 0.2.11__py3-none-any.whl → 0.3.0b3__py3-none-any.whl

xparse_client/connectors/sources/smb.py

@@ -0,0 +1,259 @@
+"""SMB/CIFS 数据源（懒加载 pysmb）"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+from io import BytesIO
+from typing import Any
+
+from ...exceptions import SourceError
+from .._utils import match_file_pattern, normalize_wildcard_patterns, to_millis_timestamp
+from .base import Source
+
+logger = logging.getLogger(__name__)
+
+
+def _get_smb_connection():
+    """懒加载 pysmb
+
+    Returns:
+        SMBConnection 类
+
+    Raises:
+        ImportError: pysmb 未安装
+    """
+    try:
+        from smb.SMBConnection import SMBConnection
+
+        return SMBConnection
+    except ImportError as e:
+        raise ImportError(
+            "使用 SmbSource 需要安装 pysmb: pip install xparse-client[smb]"
+        ) from e
+
+
+class SmbSource(Source):
+    """SMB/CIFS 数据源
+
+    从 SMB/CIFS 共享读取文件。pysmb 会在首次使用时懒加载。
+
+    Attributes:
+        host: SMB 主机
+        share_name: 共享名称
+        path: 共享内的路径
+        pattern: 文件匹配模式
+        recursive: 是否递归
+
+    Example:
+        >>> source = SmbSource(
+        ...     host="192.168.1.100",
+        ...     share_name="documents",
+        ...     username="user",
+        ...     password="pass",
+        ...     path="reports/",
+        ...     pattern=["*.pdf"],
+        ... )
+        >>> files = source.list_files()
+    """
+
+    def __init__(
+        self,
+        host: str,
+        share_name: str,
+        username: str,
+        password: str,
+        domain: str = "",
+        port: int = 445,
+        path: str = "",
+        pattern: list[str] | None = None,
+        recursive: bool = False,
+    ) -> None:
+        """初始化 SMB 数据源
+
+        Args:
+            host: SMB 主机地址
+            share_name: 共享名称
+            username: 用户名
+            password: 密码
+            domain: 域名（可选）
+            port: SMB 端口，默认 445
+            path: 共享内的路径（可选）
+            pattern: 文件匹配模式列表
+            recursive: 是否递归，默认 False
+
+        Raises:
+            SourceError: 连接失败
+        """
+        SMBConnection = _get_smb_connection()
+
+        self.host = host
+        self.share_name = share_name
+        self.username = username
+        self.domain = domain
+        self.port = port
+        self.path = path.strip("/").strip("\\") if path else ""
+        self.pattern = normalize_wildcard_patterns(pattern)
+        self.recursive = recursive
+
+        try:
+            self.conn = SMBConnection(
+                username,
+                password,
+                "",  # my_name
+                host,
+                domain=domain,
+                use_ntlm_v2=True,
+            )
+            self.conn.connect(host, port)
+            logger.info(f"SMB 连接成功: {host}/{share_name}")
+        except Exception as e:
+            raise SourceError(
+                f"SMB 连接失败: {e}",
+                connector_type="smb",
+                operation="connect",
+                details={"host": host, "share_name": share_name},
+            ) from e
+
+    def list_files(self) -> list[str]:
+        """列出 SMB 文件
+
+        Returns:
+            文件路径列表
+
+        Raises:
+            SourceError: 列出文件失败
+        """
+        try:
+            files = []
+            base_path = "/" if not self.path else f"/{self.path}"
+
+            self._list_recursive(base_path, base_path, files)
+
+            logger.info(f"SMB 找到 {len(files)} 个文件")
+            return files
+
+        except SourceError:
+            raise
+        except Exception as e:
+            raise SourceError(
+                f"列出 SMB 文件失败: {e}",
+                connector_type="smb",
+                operation="list_files",
+            ) from e
+
+    def _list_recursive(
+        self, current_path: str, base_path: str, files: list[str]
+    ) -> None:
+        """递归列出目录下的文件"""
+        try:
+            items = self.conn.listPath(self.share_name, current_path)
+
+            for item in items:
+                # 跳过 . 和 .. 以及隐藏文件
+                if item.filename in [".", ".."] or item.filename.startswith("."):
+                    continue
+
+                item_path = (
+                    f"{current_path.rstrip('/')}/{item.filename}"
+                    if current_path != "/"
+                    else f"/{item.filename}"
+                )
+
+                # 计算相对路径
+                relative_path = item_path[len(base_path) :].lstrip("/")
+
+                if item.isDirectory:
+                    if self.recursive:
+                        self._list_recursive(item_path, base_path, files)
+                    # 非递归模式忽略子目录
+                else:
+                    if match_file_pattern(relative_path, self.pattern):
+                        files.append(relative_path)
+
+        except Exception as e:
+            logger.warning(f"SMB 列出路径失败 {current_path}: {e}")
+
+    def read_file(self, file_path: str) -> tuple[bytes, dict[str, Any]]:
+        """读取 SMB 文件
+
+        Args:
+            file_path: 文件相对路径
+
+        Returns:
+            (文件内容, 元信息) 元组
+
+        Raises:
+            SourceError: 读取文件失败
+        """
+        base_path = "/" if not self.path else f"/{self.path}"
+        full_path = (
+            f"{base_path.rstrip('/')}/{file_path.lstrip('/')}"
+            if base_path != "/"
+            else f"/{file_path.lstrip('/')}"
+        )
+
+        try:
+            file_obj = BytesIO()
+            self.conn.retrieveFile(self.share_name, full_path, file_obj)
+
+            # 获取文件属性
+            date_created = None
+            date_modified = None
+            try:
+                attrs = self.conn.getAttributes(self.share_name, full_path)
+                date_created = self._to_timestamp(getattr(attrs, "create_time", None))
+                date_modified = self._to_timestamp(
+                    getattr(attrs, "last_write_time", None)
+                )
+            except Exception as e:
+                logger.debug(f"SMB 获取文件属性失败 {full_path}: {e}")
+
+            smb_url = f"smb://{self.host}/{self.share_name}{full_path}"
+            data_source = {
+                "url": smb_url,
+                "version": to_millis_timestamp(date_modified),
+                "date_created": to_millis_timestamp(date_created),
+                "date_modified": to_millis_timestamp(date_modified),
+                "record_locator": {
+                    "server": self.host,
+                    "share": self.share_name,
+                    "protocol": "smb",
+                    "remote_file_path": full_path,
+                },
+            }
+
+            file_obj.seek(0)
+            return file_obj.read(), data_source
+
+        except Exception as e:
+            raise SourceError(
+                f"读取 SMB 文件失败: {file_path}, {e}",
+                connector_type="smb",
+                operation="read_file",
+            ) from e
+
+    @staticmethod
+    def _to_timestamp(value: Any) -> float | None:
+        """将时间值转换为 Unix 时间戳"""
+        if value is None:
+            return None
+        if isinstance(value, datetime):
+            return value.astimezone(timezone.utc).timestamp()
+        if isinstance(value, (int, float)):
+            return float(value)
+        return None
+
+    def close(self) -> None:
+        """关闭 SMB 连接"""
+        try:
+            if hasattr(self, "conn") and self.conn:
+                self.conn.close()
+        except Exception:
+            pass
+
+    def __repr__(self) -> str:
+        return f"<SmbSource host={self.host} share={self.share_name}>"
+
+
+__all__ = ["SmbSource"]

xparse_client/exceptions.py

@@ -0,0 +1,398 @@
+"""统一异常类层次结构
+
+提供结构化的异常类型，便于错误处理和排查。
+
+Example:
+    >>> from xparse_client.exceptions import APIError, AuthenticationError
+    >>>
+    >>> try:
+    ...     result = client.parse.partition(file=file_bytes, filename="doc.pdf")
+    ... except AuthenticationError as e:
+    ...     print(f"认证失败: {e.message}, request_id: {e.request_id}")
+    ... except APIError as e:
+    ...     print(f"API 错误: {e}")
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class XParseClientError(Exception):
+    """xParse SDK 基础异常类
+
+    所有 SDK 异常的基类，提供统一的异常结构。
+
+    Attributes:
+        message: 错误信息
+        details: 额外的错误详情
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        self.message = message
+        self.details = details or {}
+        super().__init__(message)
+
+    def __str__(self) -> str:
+        if self.details:
+            details_str = ", ".join(f"{k}={v}" for k, v in self.details.items())
+            return f"{self.message} ({details_str})"
+        return self.message
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}(message={self.message!r}, details={self.details!r})"
+
+    def to_dict(self) -> dict[str, Any]:
+        """转换为字典，便于序列化"""
+        return {
+            "error_type": self.__class__.__name__,
+            "message": self.message,
+            "details": self.details,
+        }
+
+
+# ============================================================
+# 配置和验证错误
+# ============================================================
+
+
+class ConfigurationError(XParseClientError):
+    """配置错误
+
+    SDK 配置不正确时抛出。
+
+    Example:
+        >>> raise ConfigurationError(
+        ...     "缺少必要的配置参数",
+        ...     details={"missing_params": ["app_id", "secret_code"]}
+        ... )
+    """
+
+    pass
+
+
+class ValidationError(XParseClientError):
+    """输入验证错误
+
+    请求参数验证失败时抛出。
+
+    Attributes:
+        field: 验证失败的字段名
+        value: 传入的值
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        field: str | None = None,
+        value: Any = None,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        self.field = field
+        self.value = value
+
+        _details = details or {}
+        if field:
+            _details["field"] = field
+        if value is not None:
+            _details["value"] = str(value)[:100]  # 截断过长的值
+
+        super().__init__(message, details=_details)
+
+
+# ============================================================
+# API 错误
+# ============================================================
+
+
+class APIError(XParseClientError):
+    """API 错误基类
+
+    所有 HTTP API 相关错误的基类。
+
+    Attributes:
+        message: 错误信息
+        status_code: HTTP 状态码
+        request_id: 请求 ID（用于排查）
+        response_body: 原始响应体
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        status_code: int | None = None,
+        request_id: str | None = None,
+        response_body: str | None = None,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        self.status_code = status_code
+        self.request_id = request_id
+        self.response_body = response_body
+
+        _details = details or {}
+        if status_code:
+            _details["status_code"] = status_code
+        if request_id:
+            _details["request_id"] = request_id
+
+        super().__init__(message, details=_details)
+
+    def __str__(self) -> str:
+        parts = [self.message]
+        if self.status_code:
+            parts.append(f"status={self.status_code}")
+        if self.request_id:
+            parts.append(f"request_id={self.request_id}")
+        return " ".join(parts)
+
+
+class AuthenticationError(APIError):
+    """认证错误 (401)
+
+    API 密钥无效或已过期时抛出。
+
+    Example:
+        >>> raise AuthenticationError(
+        ...     "API 密钥无效",
+        ...     status_code=401,
+        ...     request_id="req-123"
+        ... )
+    """
+
+    pass
+
+
+class PermissionDeniedError(APIError):
+    """权限错误 (403)
+
+    无权访问请求的资源时抛出。
+    """
+
+    pass
+
+
+class NotFoundError(APIError):
+    """资源不存在错误 (404)
+
+    请求的资源不存在时抛出。
+
+    Attributes:
+        resource_type: 资源类型（如 job, workflow）
+        resource_id: 资源 ID
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        resource_type: str | None = None,
+        resource_id: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        self.resource_type = resource_type
+        self.resource_id = resource_id
+
+        details = kwargs.pop("details", {}) or {}
+        if resource_type:
+            details["resource_type"] = resource_type
+        if resource_id:
+            details["resource_id"] = resource_id
+
+        super().__init__(message, details=details, **kwargs)
+
+
+class RateLimitError(APIError):
+    """限流错误 (429)
+
+    请求频率过高时抛出。
+
+    Attributes:
+        retry_after: 建议的重试等待时间（秒）
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        retry_after: int | None = None,
+        **kwargs: Any,
+    ) -> None:
+        self.retry_after = retry_after
+
+        details = kwargs.pop("details", {}) or {}
+        if retry_after:
+            details["retry_after"] = retry_after
+
+        super().__init__(message, details=details, **kwargs)
+
+
+class ServerError(APIError):
+    """服务器错误 (5xx)
+
+    服务器内部错误时抛出。
+    """
+
+    pass
+
+
+class RequestTimeoutError(APIError):
+    """请求超时错误
+
+    请求超时时抛出。
+
+    Attributes:
+        timeout_seconds: 超时时间设置
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        timeout_seconds: float | None = None,
+        **kwargs: Any,
+    ) -> None:
+        self.timeout_seconds = timeout_seconds
+
+        details = kwargs.pop("details", {}) or {}
+        if timeout_seconds:
+            details["timeout_seconds"] = timeout_seconds
+
+        super().__init__(message, details=details, **kwargs)
+
+
+# ============================================================
+# 连接器错误
+# ============================================================
+
+
+class ConnectorError(XParseClientError):
+    """连接器错误基类
+
+    Source 和 Destination 操作相关错误的基类。
+
+    Attributes:
+        connector_type: 连接器类型（如 s3, local, milvus）
+        operation: 操作类型（如 list, read, write）
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        connector_type: str | None = None,
+        operation: str | None = None,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        self.connector_type = connector_type
+        self.operation = operation
+
+        _details = details or {}
+        if connector_type:
+            _details["connector_type"] = connector_type
+        if operation:
+            _details["operation"] = operation
+
+        super().__init__(message, details=_details)
+
+
+class SourceError(ConnectorError):
+    """数据源错误
+
+    读取数据源时发生错误。
+
+    Example:
+        >>> raise SourceError(
+        ...     "无法连接到 S3 存储桶",
+        ...     connector_type="s3",
+        ...     operation="connect",
+        ...     details={"bucket": "my-bucket"}
+        ... )
+    """
+
+    pass
+
+
+class DestinationError(ConnectorError):
+    """目的地错误
+
+    写入目的地时发生错误。
+
+    Example:
+        >>> raise DestinationError(
+        ...     "写入 Milvus 失败",
+        ...     connector_type="milvus",
+        ...     operation="write",
+        ...     details={"collection": "my_collection"}
+        ... )
+    """
+
+    pass
+
+
+# ============================================================
+# Pipeline 错误
+# ============================================================
+
+
+class PipelineError(XParseClientError):
+    """Pipeline 执行错误
+
+    Pipeline 执行过程中发生的错误。
+
+    Attributes:
+        stage: 发生错误的阶段（如 parse, chunk, embed）
+        filename: 处理的文件名
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        stage: str | None = None,
+        filename: str | None = None,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        self.stage = stage
+        self.filename = filename
+
+        _details = details or {}
+        if stage:
+            _details["stage"] = stage
+        if filename:
+            _details["filename"] = filename
+
+        super().__init__(message, details=_details)
+
+
+# ============================================================
+# 导出
+# ============================================================
+
+__all__ = [
+    # 基类
+    "XParseClientError",
+    # 配置和验证
+    "ConfigurationError",
+    "ValidationError",
+    # API 错误
+    "APIError",
+    "AuthenticationError",
+    "PermissionDeniedError",
+    "NotFoundError",
+    "RateLimitError",
+    "ServerError",
+    "RequestTimeoutError",
+    # 连接器错误
+    "ConnectorError",
+    "SourceError",
+    "DestinationError",
+    # Pipeline 错误
+    "PipelineError",
+]

xparse_client/models/__init__.py

@@ -0,0 +1,60 @@
+"""数据模型模块
+
+提供 API 请求和响应的 Pydantic 数据模型。
+"""
+
+from .chunk import ChunkConfig
+from .embed import EmbedConfig
+from .extract import ExtractConfig
+from .local import FailedFile, WorkflowResult
+from .parse import (
+    AsyncJobResponse,
+    Element,
+    ElementMetadata,
+    JobStatusResponse,
+    ParseConfig,
+    ParseResponse,
+)
+from .pipeline import (
+    ChunkStage,
+    EmbedStage,
+    ExtractStage,
+    ParseStage,
+    PipelineConfig,
+    PipelineResponse,
+    PipelineStage,
+    PipelineStats,
+)
+from .workflows import Schedule, WorkflowInformation, WorkflowState
+
+__all__ = [
+    # Parse 模型
+    "ParseConfig",
+    "Element",
+    "ElementMetadata",
+    "ParseResponse",
+    "AsyncJobResponse",
+    "JobStatusResponse",
+    # Extract 模型
+    "ExtractConfig",
+    # Chunk 模型
+    "ChunkConfig",
+    # Embed 模型
+    "EmbedConfig",
+    # Pipeline 模型
+    "ParseStage",
+    "ChunkStage",
+    "EmbedStage",
+    "ExtractStage",
+    "PipelineStage",
+    "PipelineStats",
+    "PipelineConfig",
+    "PipelineResponse",
+    # Local 模型
+    "FailedFile",
+    "WorkflowResult",
+    # Workflows 模型
+    "WorkflowInformation",
+    "WorkflowState",
+    "Schedule",
+]