xparse-client 0.2.8__py3-none-any.whl → 0.2.9__py3-none-any.whl

example/run_pipeline.py

@@ -128,14 +128,14 @@ def run_with_manual_setup():
     #     prefix='',
     #     region='cn-east-3'
     # )
-    source = S3Source(
-        endpoint='https://s3.us-east-1.amazonaws.com',
-        access_key='AKIA6QUE3TVZADUWA4PO',
-        secret_key='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',
-        bucket='textin-xparse',
-        prefix='',
-        region='us-east-1'
-    )
+    # source = S3Source(
+    #     endpoint='https://s3.us-east-1.amazonaws.com',
+    #     access_key='AKIA6QUE3TVZADUWA4PO',
+    #     secret_key='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',
+    #     bucket='textin-xparse',
+    #     prefix='',
+    #     region='us-east-1'
+    # )
     # source = S3Source(
     #     endpoint='http://127.0.0.1:9000',
     #     access_key='',
@@ -158,10 +158,11 @@ def run_with_manual_setup():
     #     username='', # 用户名，按照实际填写
     #     password=''  # 密码，按照实际填写
     # )
-    # source = LocalSource(
-    #     directory='/Users/ke_wang/Documents/doc',
-    #     pattern='*.pdf'  # 支持通配符: *.pdf, *.docx, **/*.txt
-    # )
+    source = LocalSource(
+        directory='/Users/ke_wang/Documents/doc',
+        recursive=False,
+        pattern=['*']  # 支持通配符: *.pdf, *.docx, **/*.txt
+    )
     
     # 创建 Milvus 目的地
     # destination = MilvusDestination(
@@ -194,7 +195,7 @@ def run_with_manual_setup():
     stages = [
         Stage(
             type='parse',
-            config=ParseConfig(provider='textin-lite')
+            config=ParseConfig(provider='textin')
         ),
         Stage(
             type='chunk',
@@ -236,8 +237,8 @@ def run_with_manual_setup():
     )
     
     # 运行
-    config = pipeline.get_config()
-    print(json.dumps(config, ensure_ascii=False, indent=2))
+    # config = pipeline.get_config()
+    pipeline.run()
 
 
 # ============================================================================

example/run_pipeline_test.py

@@ -86,14 +86,14 @@ def run_with_manual_setup():
     from xparse_client import ChunkConfig, EmbedConfig, ParseConfig, Stage
     
     # 创建 S3 数据源
-    # source = S3Source(
-    #     endpoint='https://textin-minio-api.ai.intsig.net',
-    #     access_key='IEQspf8C7fVcgmp3AZWl',
-    #     secret_key='kLj96I8FGbIrPFW08meXivCy4AVdzBijOJWKWOt1',
-    #     bucket='textin-test',
-    #     prefix='',
-    #     region='us-east-1'
-    # )
+    source = S3Source(
+        endpoint='https://textin-minio-api.ai.intsig.net',
+        access_key='IEQspf8C7fVcgmp3AZWl',
+        secret_key='kLj96I8FGbIrPFW08meXivCy4AVdzBijOJWKWOt1',
+        bucket='textin-test',
+        prefix='',
+        region='us-east-1'
+    )
     source = S3Source(
         endpoint='https://s3.oss-cn-shanghai.aliyuncs.com',
         access_key='LTAI5tBgsaVfkbh9rbPyuB17',

xparse_client/pipeline/sources.py

@@ -11,7 +11,7 @@ from datetime import datetime, timezone
 from email.utils import parsedate_to_datetime
 from fnmatch import fnmatch
 from pathlib import Path
-from typing import List, Dict, Any, Tuple
+from typing import List, Dict, Any, Tuple, Optional
 
 from smb.SMBConnection import SMBConnection
 from botocore.config import Config
@@ -20,6 +20,56 @@ from botocore.config import Config
 logger = logging.getLogger(__name__)
 
 
+def _normalize_wildcard_patterns(pattern: Optional[List[str]]) -> Optional[List[str]]:
+    """规范化通配符模式列表
+    
+    Args:
+        pattern: 通配符模式列表，如果为 None 或空列表则返回 None（表示匹配所有文件）
+        
+    Returns:
+        通配符模式列表，如果 pattern 是 None、空列表或包含 "*" 则返回 None（表示匹配所有文件）
+    """
+    if pattern is None or not pattern:
+        return None  # None 表示匹配所有文件
+
+    if not isinstance(pattern, list):
+        raise ValueError(f"pattern 类型错误: {type(pattern)}")
+    
+    # 过滤空字符串并去除空格
+    normalized = [p.strip() for p in pattern if p and p.strip()]
+    
+    if not normalized:
+        return None
+    
+    # 如果包含 "*"，直接返回 None（匹配所有文件，减少后续开销）
+    if '*' in normalized:
+        return None
+    
+    return normalized
+
+
+def _match_file_extension(file_path: str, wildcard_patterns: Optional[List[str]]) -> bool:
+    """检查文件路径是否匹配通配符模式
+    
+    Args:
+        file_path: 文件路径
+        wildcard_patterns: 已规范化的通配符模式列表（如 ['*.pdf', '*.docx']）
+        
+    Returns:
+        如果匹配返回 True，否则返回 False
+    """
+    # 如果 wildcard_patterns 是 None 或空列表，匹配所有文件
+    if wildcard_patterns is None:
+        return True
+    
+    # 检查是否匹配任何一个通配符模式
+    for wildcard_pattern in wildcard_patterns:
+        if fnmatch(file_path, wildcard_pattern):
+            return True
+    
+    return False
+
+
 def _to_millis_timestamp_string(timestamp):
     """将时间戳转换为毫秒时间戳字符串
     
@@ -62,11 +112,11 @@ class S3Source(Source):
     """S3/MinIO 数据源"""
 
     def __init__(self, endpoint: str, access_key: str, secret_key: str,
-                 bucket: str, prefix: str = '', region: str = 'us-east-1', pattern: str = '*', recursive: bool = False):
+                 bucket: str, prefix: str = '', region: str = 'us-east-1', pattern: Optional[List[str]] = None, recursive: bool = False):
         self.endpoint = endpoint
         self.bucket = bucket
         self.prefix = prefix
-        self.pattern = pattern or '*'
+        self.pattern = _normalize_wildcard_patterns(pattern)  # 在初始化时规范化
         self.recursive = recursive
 
         if self.endpoint == 'https://textin-minio-api.ai.intsig.net':
@@ -108,7 +158,7 @@ class S3Source(Source):
                     key = obj['Key']
                     if key.endswith('/') or key.endswith('empty.tmp'):
                         continue
-                    if fnmatch(key, self.pattern):
+                    if _match_file_extension(key, self.pattern):
                         files.append(key)
             
             # 非递归模式下，CommonPrefixes 包含子目录，我们忽略它们
@@ -156,9 +206,9 @@ class S3Source(Source):
 class LocalSource(Source):
     """本地文件系统数据源"""
 
-    def __init__(self, directory: str, pattern: str = '*', recursive: bool = False):
+    def __init__(self, directory: str, pattern: Optional[List[str]] = None, recursive: bool = False):
         self.directory = Path(directory)
-        self.pattern = pattern or '*'
+        self.pattern = _normalize_wildcard_patterns(pattern)  # 在初始化时规范化
         self.recursive = recursive
 
         if not self.directory.exists():
@@ -168,20 +218,29 @@ class LocalSource(Source):
         logger.info(f"本地目录: {self.directory}")
 
     def list_files(self) -> List[str]:
+        all_files = []
+        # 匹配所有文件
         if self.recursive:
-            # 递归模式：使用 rglob
-            files = [
+            all_files.extend([
                 str(f.relative_to(self.directory))
-                for f in self.directory.rglob(self.pattern)
+                for f in self.directory.rglob('*')
                 if f.is_file()
-            ]
+            ])
         else:
-            # 非递归模式：只列出根目录下的文件，使用 glob
-            files = [
+            all_files.extend([
                 str(f.relative_to(self.directory))
-                for f in self.directory.glob(self.pattern)
+                for f in self.directory.glob('*')
                 if f.is_file()
-            ]
+            ])
+        
+        files = []
+        if self.pattern is not None:
+            for file in all_files:
+                if _match_file_extension(file, self.pattern):
+                    files.append(file)
+        else:
+            files.extend(all_files)
+        
         print(f"✓ 本地找到 {len(files)} 个文件")
         return files
 
@@ -217,12 +276,12 @@ class LocalSource(Source):
 class FtpSource(Source):
     """FTP 数据源"""
 
-    def __init__(self, host: str, port: int, username: str, password: str, pattern: str = '*', recursive: bool = False):
+    def __init__(self, host: str, port: int, username: str, password: str, pattern: Optional[List[str]] = None, recursive: bool = False):
         self.host = host
         self.port = port
         self.username = username
         self.password = password
-        self.pattern = pattern or '*'
+        self.pattern = _normalize_wildcard_patterns(pattern)  # 在初始化时规范化
         self.recursive = recursive
 
         self.client = ftplib.FTP()
@@ -288,12 +347,12 @@ class FtpSource(Source):
                             except:
                                 # 不是目录，是文件
                                 relative_path = full_path.lstrip('/')
-                                if fnmatch(relative_path, self.pattern):
+                                if _match_file_extension(relative_path, self.pattern):
                                     files.append(relative_path)
                         else:
                             # 是文件
                             relative_path = full_path.lstrip('/')
-                            if fnmatch(relative_path, self.pattern):
+                            if _match_file_extension(relative_path, self.pattern):
                                 files.append(relative_path)
                     
                     # 恢复原始目录
@@ -325,7 +384,7 @@ class FtpSource(Source):
                     item_type = item_info.get('type', 'unknown')
                     # 只添加文件，排除目录
                     if item_type == 'file' or (item_type == 'unknown' and not item_info.get('type', '').startswith('dir')):
-                        if fnmatch(item_name, self.pattern):
+                        if _match_file_extension(item_name, self.pattern):
                             files.append(item_name)
             except:
                 # 如果不支持 MLSD，使用 LIST 命令
@@ -341,7 +400,7 @@ class FtpSource(Source):
                                 continue
                             is_dir = parts[0].startswith('d')
                             # 只添加文件，排除目录
-                            if not is_dir and fnmatch(item_name, self.pattern):
+                            if not is_dir and _match_file_extension(item_name, self.pattern):
                                 files.append(item_name)
                 except:
                     # 最后回退到 nlst，通过尝试切换目录来判断是否为目录
@@ -357,7 +416,7 @@ class FtpSource(Source):
                             continue
                         except:
                             # 不能切换，说明是文件
-                            if fnmatch(item_name, self.pattern):
+                            if _match_file_extension(item_name, self.pattern):
                                 files.append(item_name)
             
             # 确保回到原始目录
@@ -405,7 +464,7 @@ class SmbSource(Source):
     """SMB/CIFS 数据源"""
 
     def __init__(self, host: str, share_name: str, username: str, password: str,
-                 domain: str = '', port: int = 445, path: str = '', pattern: str = '*', recursive: bool = False):
+                 domain: str = '', port: int = 445, path: str = '', pattern: Optional[List[str]] = None, recursive: bool = False):
         self.host = host
         self.share_name = share_name
         self.username = username
@@ -413,7 +472,7 @@ class SmbSource(Source):
         self.domain = domain
         self.port = port
         self.path = path.strip('/').strip('\\') if path else ''
-        self.pattern = pattern or '*'
+        self.pattern = _normalize_wildcard_patterns(pattern)  # 在初始化时规范化
         self.recursive = recursive
 
         self.conn = SMBConnection(
@@ -451,7 +510,7 @@ class SmbSource(Source):
                             _list_recursive(conn, share, item_path)
                         # 非递归模式：忽略子目录
                     else:
-                        if fnmatch(relative_path, self.pattern):
+                        if _match_file_extension(relative_path, self.pattern):
                             files.append(relative_path)
             except Exception as e:
                 logger.warning(f"列出路径失败 {current_path}: {str(e)}")

{xparse_client-0.2.8.dist-info → xparse_client-0.2.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xparse-client
-Version: 0.2.8
+Version: 0.2.9
 Summary: 面向Agent和RAG的新一代文档处理 AI Infra
 License-Expression: MIT
 Project-URL: Homepage, https://gitlab.intsig.net/xparse1/xparse-pipeline
@@ -173,7 +173,7 @@ source = S3Source(
     bucket='textin',
     prefix='',
     region='us-east-1',
-    pattern='*.pdf'  # 可选，使用 Shell 通配符过滤对象
+    pattern=['*.pdf']  # 可选，通配符模式列表，支持多个扩展名
 )
 ```
 请确保配置的访问凭证至少包括以下几项权限：
@@ -193,7 +193,7 @@ source = S3Source(
     bucket='textin',
     prefix='',
     region='cn-shanghai',
-    pattern='*.pdf'  # 可选，使用 Shell 通配符过滤对象
+    pattern=['*.pdf']  # 可选，通配符模式列表，支持多个扩展名
 )
 ```
 请确保配置的访问凭证至少包括以下几项权限：
@@ -214,7 +214,7 @@ source = S3Source(
     bucket='textin',
     prefix='',
     region='ap-shanghai',
-    pattern='*.pdf'  # 可选，使用 Shell 通配符过滤对象
+    pattern=['*.pdf']  # 可选，通配符模式列表，支持多个扩展名
 )
 ```
 
@@ -235,7 +235,7 @@ source = S3Source(
     bucket='textin',
     prefix='',
     region='cn-shanghai',
-    pattern='*.pdf'  # 可选，使用 Shell 通配符过滤对象
+    pattern=['*.pdf']  # 可选，通配符模式列表，支持多个扩展名
 )
 ```
 
@@ -257,7 +257,7 @@ source = S3Source(
     bucket='textin',
     prefix='',
     region='cn-east-3',
-    pattern='*.pdf'  # 可选，使用 Shell 通配符过滤对象
+    pattern=['*.pdf']  # 可选，通配符模式列表，支持多个扩展名
 )
 ```
 
@@ -279,7 +279,7 @@ source = S3Source(
     bucket='textin-xparse',
     prefix='',
     region='us-east-1',
-    pattern='*.pdf'  # 可选，使用 Shell 通配符过滤对象
+    pattern=['*.pdf']  # 可选，通配符模式列表，支持多个扩展名
 )
 ```
 请确保配置的访问凭证至少包括以下几项权限：
@@ -294,7 +294,7 @@ s3:GetObject
 ```python
 source = LocalSource(
     directory='./input',
-    pattern='*.pdf'  # 支持通配符: *.pdf, *.docx, **/*.txt
+    pattern=['*.pdf', '*.docx']  # 支持多个通配符模式列表
 )
 ```
 
@@ -306,7 +306,7 @@ source = FtpSource(
     port=21,
     username='', # 用户名，按照实际填写
     password='',  # 密码，按照实际填写
-    pattern='*.pdf'  # 可选，过滤指定类型文件
+    pattern=['*.pdf']  # 可选，通配符模式列表，过滤指定类型文件
 )
 ```
 
@@ -319,11 +319,11 @@ source = SmbSource(
     username='',  # 用户名，按照实际填写
     password='',  # 密码，按照实际填写
     domain='your-smb-domain',
-    pattern='**/*.pdf'  # 可选，支持多级匹配
+    pattern=['**/*.pdf']  # 可选，通配符模式列表，支持多级匹配
 )
 ```
 
-> 注 1：所有 Source 均支持 `pattern` 参数，使用 Shell 通配符（`*.pdf`、`**/*.txt` 等）来过滤需要处理的文件；默认为 `*`，即处理全部文件。
+> 注 1：所有 Source 均支持 `pattern` 参数，使用通配符模式列表（如 `['*.pdf', '*.docx']`）来过滤需要处理的文件。支持多个通配符模式，如果列表中包含 `'*'` 则匹配所有文件。默认为 `None`，即处理全部文件。
 
 > 注 2：所有 Source 均支持 `recursive` 参数，表示是否递归遍历，默认为 `False`。
 
@@ -533,7 +533,7 @@ source = S3Source(
     bucket='documents',
     prefix='pdfs/',
     region='us-east-1',
-    pattern='*.pdf',  # 仅处理匹配的文件
+    pattern=['*.pdf'],  # 仅处理匹配的文件
     recursive=False   # 不递归子目录
 )
 
@@ -597,7 +597,7 @@ from xparse_client import (
 # 手动创建 Pipeline
 source = LocalSource(
     directory='./test_files',
-    pattern='*.pdf',
+    pattern=['*.pdf'],
     recursive=False
 )
 
@@ -644,7 +644,7 @@ from xparse_client import (
 # 创建本地数据源
 source = LocalSource(
     directory='./test_files',
-    pattern='*.pdf',
+    pattern=['*.pdf'],
     recursive=False
 )
 
@@ -771,7 +771,7 @@ source = FtpSource(
     port=21,
     username='user',
     password='pass',
-    pattern='*.pdf',
+    pattern=['*.pdf'],
     recursive=False
 )
 
@@ -831,7 +831,7 @@ from xparse_client import (
 # 创建 Pipeline
 source = LocalSource(
     directory='./docs',
-    pattern='*.pdf',
+    pattern=['*.pdf'],
     recursive=False
 )
 

xparse_client-0.2.9.dist-info/RECORD

@@ -0,0 +1,13 @@
+example/run_pipeline.py,sha256=xZ8TLofrK7naEwBe-tiuotcQ8yKWUES_k9iCQcIOIYo,15446
+example/run_pipeline_test.py,sha256=pxsNiq_LmP6M4R7tTuja0u-Lu7fW-wIBU1uBf0-agQI,14845
+xparse_client/__init__.py,sha256=je1ena3HwLL4CRtLU4r6EAzoOIJthlPjTwshxZnzQDM,1677
+xparse_client/pipeline/__init__.py,sha256=TVlb2AGCNKP0jrv3p4ZLZCPKp68hTVMFi00DTdi6QAo,49
+xparse_client/pipeline/config.py,sha256=FFYq2a0dBWBEj70s2aInXOiQ5MwwHimd6SI2_tkp52w,4138
+xparse_client/pipeline/destinations.py,sha256=F0z1AgVIBOn0m32i4l7LCMkJE0IbBdlpykO_at_wLaE,11931
+xparse_client/pipeline/pipeline.py,sha256=IRTxN4YUJi9Wrm1G1ysGvcwsPsGh0inbquBH3nWYmAA,26477
+xparse_client/pipeline/sources.py,sha256=D-kLrSQ-qsFFFq7JC4sL3Y3Q3Q87Wcpv9R5K85YkDjE,22144
+xparse_client-0.2.9.dist-info/licenses/LICENSE,sha256=ckIP-MbocsP9nqYnta5KgfAicYF196B5TNdHIR6kOO0,1075
+xparse_client-0.2.9.dist-info/METADATA,sha256=Faj3fvt9Fc-EW9yFDewhpkqGVo_qSvL5N-tq1aIkkyk,28086
+xparse_client-0.2.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+xparse_client-0.2.9.dist-info/top_level.txt,sha256=bfX8BWo1sEEQVsI4Ql4Uu80vrfEh5zfajU9YqFTzxMo,22
+xparse_client-0.2.9.dist-info/RECORD,,

xparse_client-0.2.8.dist-info/RECORD

@@ -1,13 +0,0 @@
-example/run_pipeline.py,sha256=ybAWBPXcQClRk1HOMySLi9IUPIs1Qn-S5HXNLbNJHjs,15459
-example/run_pipeline_test.py,sha256=uIU09FTv_VnTQS1Lc94ydc3kaD86eHkaHQbVXpsGEcA,14861
-xparse_client/__init__.py,sha256=je1ena3HwLL4CRtLU4r6EAzoOIJthlPjTwshxZnzQDM,1677
-xparse_client/pipeline/__init__.py,sha256=TVlb2AGCNKP0jrv3p4ZLZCPKp68hTVMFi00DTdi6QAo,49
-xparse_client/pipeline/config.py,sha256=FFYq2a0dBWBEj70s2aInXOiQ5MwwHimd6SI2_tkp52w,4138
-xparse_client/pipeline/destinations.py,sha256=F0z1AgVIBOn0m32i4l7LCMkJE0IbBdlpykO_at_wLaE,11931
-xparse_client/pipeline/pipeline.py,sha256=IRTxN4YUJi9Wrm1G1ysGvcwsPsGh0inbquBH3nWYmAA,26477
-xparse_client/pipeline/sources.py,sha256=UeVbWv6n0wQkIZIBBhrFCiyydQX7cvwmkoMgcf12p9g,19940
-xparse_client-0.2.8.dist-info/licenses/LICENSE,sha256=ckIP-MbocsP9nqYnta5KgfAicYF196B5TNdHIR6kOO0,1075
-xparse_client-0.2.8.dist-info/METADATA,sha256=LX8TfLSbFZerGPhh16x5QK1lwrPh55CYKNLhr2kdBcY,27850
-xparse_client-0.2.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-xparse_client-0.2.8.dist-info/top_level.txt,sha256=bfX8BWo1sEEQVsI4Ql4Uu80vrfEh5zfajU9YqFTzxMo,22
-xparse_client-0.2.8.dist-info/RECORD,,