xparse-client 0.2.20__py3-none-any.whl → 0.3.0b1__py3-none-any.whl

xparse_client/connectors/sources/ftp.py

@@ -0,0 +1,278 @@
+"""FTP 数据源"""
+
+from __future__ import annotations
+
+import ftplib
+import logging
+from datetime import datetime, timezone
+from io import BytesIO
+from typing import Any
+
+from ...exceptions import SourceError
+from .._utils import match_file_pattern, normalize_wildcard_patterns, to_millis_timestamp
+from .base import Source
+
+logger = logging.getLogger(__name__)
+
+
+class FtpSource(Source):
+    """FTP 数据源
+
+    从 FTP 服务器读取文件。
+
+    Attributes:
+        host: FTP 主机
+        port: FTP 端口
+        username: 用户名
+        pattern: 文件匹配模式
+        recursive: 是否递归
+
+    Example:
+        >>> source = FtpSource(
+        ...     host="ftp.example.com",
+        ...     port=21,
+        ...     username="user",
+        ...     password="pass",
+        ...     pattern=["*.pdf"],
+        ... )
+        >>> files = source.list_files()
+    """
+
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        username: str,
+        password: str,
+        pattern: list[str] | None = None,
+        recursive: bool = False,
+    ) -> None:
+        """初始化 FTP 数据源
+
+        Args:
+            host: FTP 主机地址
+            port: FTP 端口
+            username: 用户名
+            password: 密码
+            pattern: 文件匹配模式列表
+            recursive: 是否递归，默认 False
+
+        Raises:
+            SourceError: 连接失败
+        """
+        self.host = host
+        self.port = port
+        self.username = username
+        self.password = password
+        self.pattern = normalize_wildcard_patterns(pattern)
+        self.recursive = recursive
+
+        try:
+            self.client = ftplib.FTP()
+            self.client.connect(self.host, self.port)
+            self.client.login(self.username, self.password)
+            logger.info(f"FTP 连接成功: {host}:{port}")
+        except Exception as e:
+            raise SourceError(
+                f"FTP 连接失败: {e}",
+                connector_type="ftp",
+                operation="connect",
+                details={"host": host, "port": port},
+            ) from e
+
+    def list_files(self) -> list[str]:
+        """列出 FTP 文件
+
+        Returns:
+            文件路径列表
+
+        Raises:
+            SourceError: 列出文件失败
+        """
+        try:
+            files = []
+            current_dir = self.client.pwd()
+
+            if self.recursive:
+                self._list_recursive("", files)
+            else:
+                self._list_current_dir(files)
+
+            # 确保回到原始目录
+            try:
+                self.client.cwd(current_dir)
+            except Exception:
+                pass
+
+            logger.info(f"FTP 找到 {len(files)} 个文件")
+            return files
+
+        except SourceError:
+            raise
+        except Exception as e:
+            raise SourceError(
+                f"列出 FTP 文件失败: {e}",
+                connector_type="ftp",
+                operation="list_files",
+            ) from e
+
+    def _list_recursive(self, path: str, files: list[str]) -> None:
+        """递归列出目录下的文件"""
+        try:
+            original_dir = self.client.pwd()
+            if path:
+                try:
+                    self.client.cwd(path)
+                except Exception:
+                    return
+
+            items = self._get_dir_items()
+
+            for item_name, is_dir in items:
+                if item_name in [".", ".."]:
+                    continue
+
+                full_path = f"{path}/{item_name}" if path else item_name
+
+                if is_dir:
+                    self._list_recursive(full_path, files)
+                else:
+                    relative_path = full_path.lstrip("/")
+                    if match_file_pattern(relative_path, self.pattern):
+                        files.append(relative_path)
+
+            self.client.cwd(original_dir)
+
+        except Exception as e:
+            logger.warning(f"FTP 列出路径失败 {path}: {e}")
+
+    def _list_current_dir(self, files: list[str]) -> None:
+        """列出当前目录下的文件（非递归）"""
+        items = self._get_dir_items()
+
+        for item_name, is_dir in items:
+            if item_name in [".", ".."] or is_dir:
+                continue
+
+            if match_file_pattern(item_name, self.pattern):
+                files.append(item_name)
+
+    def _get_dir_items(self) -> list[tuple[str, bool]]:
+        """获取当前目录的项目列表
+
+        Returns:
+            (文件名, 是否为目录) 元组列表
+        """
+        items = []
+
+        # 尝试使用 MLSD（更可靠）
+        try:
+            for item_name, item_info in self.client.mlsd():
+                if item_name in [".", ".."]:
+                    continue
+                is_dir = item_info.get("type") == "dir"
+                items.append((item_name, is_dir))
+            return items
+        except Exception:
+            pass
+
+        # 回退到 LIST 命令
+        try:
+            lines = []
+            self.client.retrlines("LIST", lines.append)
+            for line in lines:
+                parts = line.split()
+                if len(parts) >= 9:
+                    item_name = " ".join(parts[8:])
+                    if item_name in [".", ".."]:
+                        continue
+                    is_dir = parts[0].startswith("d")
+                    items.append((item_name, is_dir))
+            return items
+        except Exception:
+            pass
+
+        # 最后回退到 NLST，通过尝试切换目录判断类型
+        try:
+            raw_items = self.client.nlst()
+            for item_name in raw_items:
+                if item_name in [".", ".."]:
+                    continue
+                try:
+                    self.client.cwd(item_name)
+                    self.client.cwd("..")
+                    is_dir = True
+                except Exception:
+                    is_dir = False
+                items.append((item_name, is_dir))
+        except Exception:
+            pass
+
+        return items
+
+    def read_file(self, file_path: str) -> tuple[bytes, dict[str, Any]]:
+        """读取 FTP 文件
+
+        Args:
+            file_path: 文件路径
+
+        Returns:
+            (文件内容, 元信息) 元组
+
+        Raises:
+            SourceError: 读取文件失败
+        """
+        try:
+            buffer = BytesIO()
+            self.client.retrbinary(f"RETR {file_path}", buffer.write)
+
+            # 获取修改时间
+            date_modified = None
+            try:
+                resp = self.client.sendcmd(f"MDTM {file_path}")
+                parts = resp.split()
+                if len(parts) == 2 and parts[0] == "213":
+                    dt = datetime.strptime(parts[1], "%Y%m%d%H%M%S")
+                    date_modified = dt.replace(tzinfo=timezone.utc).timestamp()
+            except Exception as e:
+                logger.debug(f"FTP 获取文件时间失败 {file_path}: {e}")
+
+            normalized_path = file_path.lstrip("/")
+            version = to_millis_timestamp(date_modified)
+
+            data_source = {
+                "url": f"ftp://{self.host}:{self.port}/{normalized_path}",
+                "version": version,
+                "date_created": version,
+                "date_modified": version,
+                "record_locator": {
+                    "server": f"{self.host}:{self.port}",
+                    "protocol": "ftp",
+                    "remote_file_path": normalized_path,
+                },
+            }
+
+            return buffer.getvalue(), data_source
+
+        except Exception as e:
+            raise SourceError(
+                f"读取 FTP 文件失败: {file_path}, {e}",
+                connector_type="ftp",
+                operation="read_file",
+            ) from e
+
+    def close(self) -> None:
+        """关闭 FTP 连接"""
+        try:
+            self.client.quit()
+        except Exception:
+            try:
+                self.client.close()
+            except Exception:
+                pass
+
+    def __repr__(self) -> str:
+        return f"<FtpSource host={self.host}:{self.port}>"
+
+
+__all__ = ["FtpSource"]

xparse_client/connectors/sources/local.py

@@ -0,0 +1,176 @@
+"""本地文件系统数据源"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from ...exceptions import SourceError
+from .._utils import match_file_pattern, normalize_wildcard_patterns, to_millis_timestamp
+from .base import Source
+
+logger = logging.getLogger(__name__)
+
+
+class LocalSource(Source):
+    """本地文件系统数据源
+
+    从本地目录读取文件。
+
+    Attributes:
+        directory: 目录路径
+        pattern: 文件匹配模式
+        recursive: 是否递归子目录
+
+    Example:
+        >>> source = LocalSource(
+        ...     directory="./documents",
+        ...     pattern=["*.pdf", "*.docx"],
+        ...     recursive=True,
+        ... )
+        >>> files = source.list_files()
+        >>> content, metadata = source.read_file(files[0])
+    """
+
+    def __init__(
+        self,
+        directory: str,
+        pattern: list[str] | None = None,
+        recursive: bool = False,
+    ) -> None:
+        """初始化本地数据源
+
+        Args:
+            directory: 目录路径
+            pattern: 文件匹配模式列表，如 ["*.pdf", "*.docx"]
+            recursive: 是否递归子目录，默认 False
+
+        Raises:
+            SourceError: 目录不存在
+        """
+        self.directory = Path(directory).resolve()
+        self.pattern = normalize_wildcard_patterns(pattern)
+        self.recursive = recursive
+
+        if not self.directory.exists():
+            raise SourceError(
+                f"目录不存在: {directory}",
+                connector_type="local",
+                operation="init",
+            )
+
+        if not self.directory.is_dir():
+            raise SourceError(
+                f"路径不是目录: {directory}",
+                connector_type="local",
+                operation="init",
+            )
+
+        logger.info(f"本地数据源初始化: {self.directory}")
+
+    def list_files(self) -> list[str]:
+        """列出目录下的文件
+
+        Returns:
+            文件相对路径列表
+
+        Raises:
+            SourceError: 列出文件失败
+        """
+        try:
+            glob_pattern = "**/*" if self.recursive else "*"
+            all_files = []
+
+            for f in self.directory.glob(glob_pattern):
+                if f.is_file():
+                    # 返回相对路径
+                    rel_path = str(f.relative_to(self.directory))
+                    all_files.append(rel_path)
+
+            # 按模式过滤
+            files = [f for f in all_files if match_file_pattern(f, self.pattern)]
+
+            logger.info(f"本地数据源找到 {len(files)} 个文件")
+            return sorted(files)
+
+        except SourceError:
+            raise
+        except Exception as e:
+            raise SourceError(
+                f"列出文件失败: {e}",
+                connector_type="local",
+                operation="list_files",
+            ) from e
+
+    def read_file(self, file_path: str) -> tuple[bytes, dict[str, Any]]:
+        """读取本地文件
+
+        Args:
+            file_path: 文件相对路径
+
+        Returns:
+            (文件内容, 元信息) 元组
+
+        Raises:
+            SourceError: 读取文件失败
+        """
+        full_path = (self.directory / file_path).resolve()
+
+        # 安全检查：确保路径在目录内
+        try:
+            full_path.relative_to(self.directory)
+        except ValueError as e:
+            raise SourceError(
+                f"非法路径，不在目录内: {file_path}",
+                connector_type="local",
+                operation="read_file",
+            ) from e
+
+        try:
+            with open(full_path, "rb") as f:
+                file_bytes = f.read()
+
+            # 获取文件元信息
+            stats = full_path.stat()
+            date_created = stats.st_ctime
+            date_modified = stats.st_mtime
+            version = str(int(stats.st_mtime_ns))
+
+            data_source = {
+                "url": full_path.as_uri(),
+                "version": version,
+                "date_created": to_millis_timestamp(date_created),
+                "date_modified": to_millis_timestamp(date_modified),
+                "record_locator": {
+                    "protocol": "file",
+                    "remote_file_path": str(full_path),
+                },
+            }
+
+            return file_bytes, data_source
+
+        except FileNotFoundError as e:
+            raise SourceError(
+                f"文件不存在: {full_path}",
+                connector_type="local",
+                operation="read_file",
+            ) from e
+        except PermissionError as e:
+            raise SourceError(
+                f"无权限读取文件: {full_path}",
+                connector_type="local",
+                operation="read_file",
+            ) from e
+        except Exception as e:
+            raise SourceError(
+                f"读取文件失败: {e}",
+                connector_type="local",
+                operation="read_file",
+            ) from e
+
+    def __repr__(self) -> str:
+        return f"<LocalSource directory={self.directory}>"
+
+
+__all__ = ["LocalSource"]

xparse_client/connectors/sources/s3.py

@@ -0,0 +1,232 @@
+"""S3/MinIO 数据源（懒加载 boto3）"""
+
+from __future__ import annotations
+
+import logging
+from datetime import timezone
+from email.utils import parsedate_to_datetime
+from typing import Any
+
+from ...exceptions import SourceError
+from .._utils import match_file_pattern, normalize_wildcard_patterns, to_millis_timestamp
+from .base import Source
+
+logger = logging.getLogger(__name__)
+
+
+def _get_boto3():
+    """懒加载 boto3
+
+    Returns:
+        (boto3, Config) 元组
+
+    Raises:
+        ImportError: boto3 未安装
+    """
+    try:
+        import boto3
+        from botocore.config import Config
+
+        return boto3, Config
+    except ImportError as e:
+        raise ImportError(
+            "使用 S3Source 需要安装 boto3: pip install xparse-client[s3]"
+        ) from e
+
+
+class S3Source(Source):
+    """S3/MinIO 数据源
+
+    从 S3 或兼容存储（MinIO、阿里云 OSS、华为云 OBS 等）读取文件。
+    boto3 会在首次使用时懒加载。
+
+    Attributes:
+        endpoint: S3 端点 URL
+        bucket: 存储桶名称
+        prefix: 对象前缀
+        pattern: 文件匹配模式
+        recursive: 是否递归
+
+    Example:
+        >>> source = S3Source(
+        ...     endpoint="https://s3.amazonaws.com",
+        ...     access_key="your-access-key",
+        ...     secret_key="your-secret-key",
+        ...     bucket="my-bucket",
+        ...     prefix="documents/",
+        ...     pattern=["*.pdf"],
+        ... )
+        >>> files = source.list_files()
+        >>> content, metadata = source.read_file(files[0])
+    """
+
+    def __init__(
+        self,
+        endpoint: str,
+        access_key: str,
+        secret_key: str,
+        bucket: str,
+        prefix: str = "",
+        region: str = "us-east-1",
+        pattern: list[str] | None = None,
+        recursive: bool = False,
+    ) -> None:
+        """初始化 S3 数据源
+
+        Args:
+            endpoint: S3 端点 URL
+            access_key: 访问密钥
+            secret_key: 秘密密钥
+            bucket: 存储桶名称
+            prefix: 对象前缀（可选）
+            region: 区域，默认 us-east-1
+            pattern: 文件匹配模式列表
+            recursive: 是否递归，默认 False
+
+        Raises:
+            SourceError: 连接失败
+        """
+        boto3, Config = _get_boto3()
+
+        self.endpoint = endpoint
+        self.bucket = bucket
+        self.prefix = prefix
+        self.pattern = normalize_wildcard_patterns(pattern)
+        self.recursive = recursive
+
+        # 根据端点配置签名版本
+        if endpoint == "https://textin-minio-api.ai.intsig.net":
+            config = Config(signature_version="s3v4")
+        elif endpoint.endswith("aliyuncs.com"):
+            config = Config(signature_version="s3", s3={"addressing_style": "virtual"})
+        elif endpoint.endswith("myhuaweicloud.com"):
+            config = Config(signature_version="s3", s3={"addressing_style": "virtual"})
+        else:
+            config = Config(signature_version="s3v4", s3={"addressing_style": "virtual"})
+
+        self.client = boto3.client(
+            "s3",
+            endpoint_url=endpoint,
+            aws_access_key_id=access_key,
+            aws_secret_access_key=secret_key,
+            region_name=region,
+            config=config,
+        )
+
+        # 验证连接
+        try:
+            self.client.head_bucket(Bucket=bucket)
+            logger.info(f"S3 连接成功: {endpoint}/{bucket}")
+        except Exception as e:
+            raise SourceError(
+                f"S3 连接失败: {e}",
+                connector_type="s3",
+                operation="connect",
+                details={"endpoint": endpoint, "bucket": bucket},
+            ) from e
+
+    def list_files(self) -> list[str]:
+        """列出 S3 对象
+
+        Returns:
+            对象 Key 列表
+
+        Raises:
+            SourceError: 列出对象失败
+        """
+        try:
+            files = []
+            paginator = self.client.get_paginator("list_objects_v2")
+
+            params = {"Bucket": self.bucket}
+            if self.prefix:
+                params["Prefix"] = self.prefix
+            if not self.recursive:
+                # 非递归模式使用 Delimiter
+                params["Delimiter"] = "/"
+
+            for page in paginator.paginate(**params):
+                if "Contents" in page:
+                    for obj in page["Contents"]:
+                        key = obj["Key"]
+                        # 跳过目录和临时文件
+                        if key.endswith("/") or key.endswith("empty.tmp"):
+                            continue
+                        if match_file_pattern(key, self.pattern):
+                            files.append(key)
+
+            logger.info(f"S3 找到 {len(files)} 个文件")
+            return files
+
+        except SourceError:
+            raise
+        except Exception as e:
+            raise SourceError(
+                f"列出 S3 对象失败: {e}",
+                connector_type="s3",
+                operation="list_files",
+            ) from e
+
+    def read_file(self, file_path: str) -> tuple[bytes, dict[str, Any]]:
+        """读取 S3 对象
+
+        Args:
+            file_path: 对象 Key
+
+        Returns:
+            (对象内容, 元信息) 元组
+
+        Raises:
+            SourceError: 读取对象失败
+        """
+        try:
+            response = self.client.get_object(Bucket=self.bucket, Key=file_path)
+            file_bytes = response["Body"].read()
+
+            # 获取元信息
+            headers = response.get("ResponseMetadata", {}).get("HTTPHeaders", {})
+            version = headers.get("etag", "").strip('"')
+            last_modified = headers.get("last-modified")
+            server = headers.get("server", "unknown")
+
+            date_modified = None
+            if last_modified:
+                try:
+                    dt = parsedate_to_datetime(last_modified)
+                    date_modified = dt.astimezone(timezone.utc).timestamp()
+                except Exception:
+                    pass
+
+            normalized_key = file_path.lstrip("/")
+            data_source = {
+                "url": f"s3://{self.bucket}/{normalized_key}",
+                "version": version,
+                "date_created": to_millis_timestamp(date_modified),
+                "date_modified": to_millis_timestamp(date_modified),
+                "record_locator": {
+                    "server": server,
+                    "protocol": "s3",
+                    "remote_file_path": normalized_key,
+                },
+            }
+
+            return file_bytes, data_source
+
+        except self.client.exceptions.NoSuchKey as e:
+            raise SourceError(
+                f"S3 对象不存在: {file_path}",
+                connector_type="s3",
+                operation="read_file",
+            ) from e
+        except Exception as e:
+            raise SourceError(
+                f"读取 S3 对象失败: {file_path}, {e}",
+                connector_type="s3",
+                operation="read_file",
+            ) from e
+
+    def __repr__(self) -> str:
+        return f"<S3Source endpoint={self.endpoint} bucket={self.bucket}>"
+
+
+__all__ = ["S3Source"]