xparse-client 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl

xparse_client/pipeline/sources.py

@@ -0,0 +1,342 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+
+import json
+import logging
+import boto3
+import ftplib
+
+from abc import ABC, abstractmethod
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+from fnmatch import fnmatch
+from pathlib import Path
+from typing import List, Dict, Any, Tuple
+
+from smb.SMBConnection import SMBConnection
+from botocore.config import Config
+
+
+logger = logging.getLogger(__name__)
+
+
+class Source(ABC):
+    """数据源抽象基类"""
+
+    @abstractmethod
+    def list_files(self) -> List[str]:
+        """列出所有文件"""
+        raise NotImplementedError
+
+    @abstractmethod
+    def read_file(self, file_path: str) -> Tuple[bytes, Dict[str, Any]]:
+        """读取文件内容并返回数据来源信息"""
+        raise NotImplementedError
+
+
+class S3Source(Source):
+    """S3/MinIO 数据源"""
+
+    def __init__(self, endpoint: str, access_key: str, secret_key: str,
+                 bucket: str, prefix: str = '', region: str = 'us-east-1', pattern: str = '*'):
+        self.endpoint = endpoint
+        self.bucket = bucket
+        self.prefix = prefix
+        self.pattern = pattern or '*'
+
+        if self.endpoint == 'https://textin-minio-api.ai.intsig.net':
+            config = Config(signature_version='s3v4')
+        else:
+            config = Config(signature_version='s3v4', s3={'addressing_style': 'virtual'})
+
+        self.client = boto3.client(
+            's3',
+            endpoint_url=endpoint,
+            aws_access_key_id=access_key,
+            aws_secret_access_key=secret_key,
+            region_name=region,
+            config=config
+        )
+
+        try:
+            self.client.head_bucket(Bucket=bucket)
+            print(f"✓ S3 连接成功: {endpoint}/{bucket}")
+            logger.info(f"S3 连接成功: {endpoint}/{bucket}")
+        except Exception as e:
+            print(f"✗ S3 连接失败: {str(e)}")
+            raise
+
+    def list_files(self) -> List[str]:
+        files = []
+        paginator = self.client.get_paginator('list_objects_v2')
+
+        params = {'Bucket': self.bucket}
+        if self.prefix:
+            params['Prefix'] = self.prefix
+
+        for page in paginator.paginate(**params):
+            if 'Contents' in page:
+                for obj in page['Contents']:
+                    key = obj['Key']
+                    if key.endswith('/') or key.endswith('empty.tmp'):
+                        continue
+                    if fnmatch(key, self.pattern):
+                        files.append(key)
+
+        print(f"✓ S3 找到 {len(files)} 个文件")
+        return files
+
+    def read_file(self, file_path: str) -> Tuple[bytes, Dict[str, Any]]:
+        response = self.client.get_object(Bucket=self.bucket, Key=file_path)
+        file_bytes = response['Body'].read()
+
+        headers = response.get('ResponseMetadata', {}).get('HTTPHeaders', {})
+        version = response.get('VersionId') or headers.get('x-amz-version-id')
+        last_modified = headers.get('last-modified')
+        server = headers.get('server') or "unknown"
+        date_modified = None
+        if last_modified:
+            try:
+                dt = parsedate_to_datetime(last_modified)
+                date_modified = dt.astimezone(timezone.utc).timestamp()
+            except Exception as exc:
+                logger.debug(f"S3 解析 last-modified 失败 {file_path}: {exc}")
+
+        normalized_key = file_path.lstrip('/')
+        data_source = {
+            'url': f"s3://{self.bucket}/{normalized_key}",
+            'version': version,
+            'date_created': date_modified,
+            'date_modified': date_modified,
+            'record_locator': {
+                'server': server,
+                'protocol': 's3',
+                'remote_file_path': normalized_key
+            }
+        }
+        
+        return file_bytes, data_source
+
+
+class LocalSource(Source):
+    """本地文件系统数据源"""
+
+    def __init__(self, directory: str, pattern: str = '*'):
+        self.directory = Path(directory)
+        self.pattern = pattern or '*'
+
+        if not self.directory.exists():
+            raise ValueError(f"目录不存在: {directory}")
+
+        print(f"✓ 本地目录: {self.directory}")
+        logger.info(f"本地目录: {self.directory}")
+
+    def list_files(self) -> List[str]:
+        files = [
+            str(f.relative_to(self.directory))
+            for f in self.directory.rglob(self.pattern)
+            if f.is_file()
+        ]
+        print(f"✓ 本地找到 {len(files)} 个文件")
+        return files
+
+    def read_file(self, file_path: str) -> Tuple[bytes, Dict[str, Any]]:
+        full_path = (self.directory / file_path).resolve()
+        with open(full_path, 'rb') as f:
+            file_bytes = f.read()
+
+        date_created = None
+        date_modified = None
+        version = None
+        try:
+            stats = full_path.stat()
+            date_created = stats.st_ctime
+            date_modified = stats.st_mtime
+            version = str(int(stats.st_mtime_ns))
+        except FileNotFoundError:
+            logger.warning(f"本地文件不存在，无法获取 metadata: {full_path}")
+
+        data_source = {
+            'url': full_path.as_uri(),
+            'version': version,
+            'date_created': date_created,
+            'date_modified': date_modified,
+            'record_locator': {
+                'protocol': 'file',
+                'remote_file_path': str(full_path)
+            }
+        }
+        return file_bytes, data_source
+
+
+class FtpSource(Source):
+    """FTP 数据源"""
+
+    def __init__(self, host: str, port: int, username: str, password: str, pattern: str = '*'):
+        self.host = host
+        self.port = port
+        self.username = username
+        self.password = password
+        self.pattern = pattern or '*'
+
+        self.client = ftplib.FTP()
+        self.client.connect(self.host, self.port)
+        self.client.login(self.username, self.password)
+
+        print(f"✓ FTP 连接成功: {self.host}:{self.port}")
+        logger.info(f"FTP 连接成功: {self.host}:{self.port}")
+
+    def list_files(self) -> List[str]:
+        raw_files = self.client.nlst()
+        files = [f for f in raw_files if fnmatch(f, self.pattern)]
+        print(f"✓ FTP 找到 {len(files)} 个文件 (匹配 pattern)")
+        return files
+
+    def read_file(self, file_path: str) -> Tuple[bytes, Dict[str, Any]]:
+        from io import BytesIO
+        buffer = BytesIO()
+        self.client.retrbinary(f'RETR {file_path}', buffer.write)
+
+        date_modified = None
+        try:
+            resp = self.client.sendcmd(f"MDTM {file_path}")
+            parts = resp.split()
+            if len(parts) == 2 and parts[0] == '213':
+                dt = datetime.strptime(parts[1], "%Y%m%d%H%M%S")
+                date_modified = dt.replace(tzinfo=timezone.utc).timestamp()
+        except Exception as exc:
+            logger.debug(f"FTP 获取文件时间失败 {file_path}: {exc}")
+
+        normalized_path = file_path.lstrip('/')
+        data_source = {
+            'url': f"ftp://{self.host}:{self.port}/{normalized_path}",
+            'version': None,
+            'date_created': None,
+            'date_modified': date_modified,
+            'record_locator': {
+                'server': f"{self.host}:{self.port}",
+                'protocol': 'ftp',
+                'remote_file_path': normalized_path
+            }
+        }
+
+        return buffer.getvalue(), data_source
+
+
+class SmbSource(Source):
+    """SMB/CIFS 数据源"""
+
+    def __init__(self, host: str, share_name: str, username: str, password: str,
+                 domain: str = '', port: int = 445, path: str = '', pattern: str = '*'):
+        self.host = host
+        self.share_name = share_name
+        self.username = username
+        self.password = password
+        self.domain = domain
+        self.port = port
+        self.path = path.strip('/').strip('\\') if path else ''
+        self.pattern = pattern or '*'
+
+        self.conn = SMBConnection(
+            username,
+            password,
+            '',
+            host,
+            domain=domain,
+            use_ntlm_v2=True
+        )
+
+        try:
+            self.conn.connect(host, port)
+        except Exception as e:
+            error_msg = f"无法连接到 SMB 服务器 {host}:{port}: {str(e)}"
+            print(f"✗ SMB 连接失败: {error_msg}")
+            logger.error(f"SMB 连接失败: {error_msg}")
+            raise ConnectionError(error_msg)
+
+    def list_files(self) -> List[str]:
+        files = []
+        base_path = '/' if not self.path else f'/{self.path}'
+
+        def _list_recursive(conn, share, current_path):
+            try:
+                items = conn.listPath(share, current_path)
+                for item in items:
+                    if item.filename in ['.', '..'] or item.filename.startswith('.'):
+                        continue
+                    item_path = f"{current_path.rstrip('/')}/{item.filename}" if current_path != '/' else f"/{item.filename}"
+                    relative_path = item_path[len(base_path):].lstrip('/')
+                    if item.isDirectory:
+                        _list_recursive(conn, share, item_path)
+                    else:
+                        if fnmatch(relative_path, self.pattern):
+                            files.append(relative_path)
+            except Exception as e:
+                logger.warning(f"列出路径失败 {current_path}: {str(e)}")
+
+        _list_recursive(self.conn, self.share_name, base_path)
+
+        print(f"✓ SMB 找到 {len(files)} 个文件")
+        return files
+
+    def read_file(self, file_path: str) -> Tuple[bytes, Dict[str, Any]]:
+        from io import BytesIO
+
+        base_path = '/' if not self.path else f'/{self.path}'
+        full_path = f"{base_path.rstrip('/')}/{file_path.lstrip('/')}" if base_path != '/' else f"/{file_path.lstrip('/')}"
+
+        file_obj = BytesIO()
+        try:
+            self.conn.retrieveFile(self.share_name, full_path, file_obj)
+        except Exception as e:
+            raise IOError(f"读取文件失败 {full_path}: {str(e)}")
+
+        def _to_timestamp(value):
+            if isinstance(value, datetime):
+                return value.astimezone(timezone.utc).timestamp()
+            if isinstance(value, (int, float)):
+                return value
+            return None
+
+        date_created = None
+        date_modified = None
+        try:
+            attrs = self.conn.getAttributes(self.share_name, full_path)
+            date_created = _to_timestamp(getattr(attrs, 'create_time', None))
+            date_modified = _to_timestamp(getattr(attrs, 'last_write_time', None))
+        except Exception as exc:
+            logger.debug(f"SMB 获取文件属性失败 {full_path}: {exc}")
+
+        smb_url = f"smb://{self.host}/{self.share_name}{full_path}"
+        data_source = {
+            'url': smb_url,
+            'version': None,
+            'date_created': date_created,
+            'date_modified': date_modified,
+            'record_locator': {
+                'server': self.host,
+                'share': self.share_name,
+                'protocol': 'smb',
+                'remote_file_path': full_path
+            }
+        }
+
+        file_obj.seek(0)
+        return file_obj.read(), data_source
+
+    def __del__(self):
+        if hasattr(self, 'conn') and self.conn:
+            try:
+                self.conn.close()
+            except Exception:
+                pass
+
+
+__all__ = [
+    'Source',
+    'S3Source',
+    'LocalSource',
+    'FtpSource',
+    'SmbSource',
+]
+

{xparse_client-0.2.2.dist-info → xparse_client-0.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xparse-client
-Version: 0.2.2
+Version: 0.2.4
 Summary: 面向Agent和RAG的新一代文档处理 AI Infra
 License-Expression: MIT
 Project-URL: Homepage, https://gitlab.intsig.net/xparse1/xparse-pipeline

xparse_client-0.2.4.dist-info/RECORD

@@ -0,0 +1,13 @@
+example/run_pipeline.py,sha256=6gavTizAIqD62g4n9Pjq2-yW57ItZMJOOw8GEKm0Byk,15125
+example/run_pipeline_test.py,sha256=uIU09FTv_VnTQS1Lc94ydc3kaD86eHkaHQbVXpsGEcA,14861
+xparse_client/__init__.py,sha256=je1ena3HwLL4CRtLU4r6EAzoOIJthlPjTwshxZnzQDM,1677
+xparse_client/pipeline/__init__.py,sha256=TVlb2AGCNKP0jrv3p4ZLZCPKp68hTVMFi00DTdi6QAo,49
+xparse_client/pipeline/config.py,sha256=gkhAF-55PNvPPyfTZ0HkP95XB_K0HKCyYl6R4PTQLhI,4045
+xparse_client/pipeline/destinations.py,sha256=rqcxmsn1YGClVxGQxSVmyr-uumOVilOv_vX82fUBj-I,9859
+xparse_client/pipeline/pipeline.py,sha256=oz_BKWLbslkuRsxG0zEfh9url7saLWgtoTH1mrK6gCc,18282
+xparse_client/pipeline/sources.py,sha256=-0Eutg9t8xni12cfv2bdQVdImlkCQ7gWlOXIFBt6tpE,11568
+xparse_client-0.2.4.dist-info/licenses/LICENSE,sha256=ckIP-MbocsP9nqYnta5KgfAicYF196B5TNdHIR6kOO0,1075
+xparse_client-0.2.4.dist-info/METADATA,sha256=IMgXO9a7wnN0Ygzauk7eOkyrRFE3A2rq73eofvq3wBs,26508
+xparse_client-0.2.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+xparse_client-0.2.4.dist-info/top_level.txt,sha256=bfX8BWo1sEEQVsI4Ql4Uu80vrfEh5zfajU9YqFTzxMo,22
+xparse_client-0.2.4.dist-info/RECORD,,

{xparse_client-0.2.2.dist-info → xparse_client-0.2.4.dist-info}/top_level.txt

@@ -1 +1,2 @@
+example
 xparse_client

xparse_client-0.2.2.dist-info/RECORD

@@ -1,6 +0,0 @@
-xparse_client/__init__.py,sha256=3c_nFCooim4J31P0QWPM2VqdFJiHNFdH44IRmIIHvjk,901
-xparse_client-0.2.2.dist-info/licenses/LICENSE,sha256=ckIP-MbocsP9nqYnta5KgfAicYF196B5TNdHIR6kOO0,1075
-xparse_client-0.2.2.dist-info/METADATA,sha256=F390ItwP4PGN-9FvJGOgqjtX7YWgMJolSJnxqbEBq08,26508
-xparse_client-0.2.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-xparse_client-0.2.2.dist-info/top_level.txt,sha256=W5PeQwOyfo_Od3d26-gcOtan7rHYk1q3SP1phYedat4,14
-xparse_client-0.2.2.dist-info/RECORD,,