xparse-client 0.2.3__py3-none-any.whl → 0.2.5__py3-none-any.whl

example/run_pipeline.py

@@ -15,8 +15,8 @@ from xparse_client import create_pipeline_from_config, S3Source, LocalSource, Mi
 
 # API 请求头配置
 API_HEADERS = {
-    'x-ti-app-id': '',
-    'x-ti-secret-code': ''
+    'x-ti-app-id': '4c0032d9e4d93b0ad674cac0d75256e7',
+    'x-ti-secret-code': '7104f599ad02b8468fc619f7605d2d8d'
 }
 
 
@@ -168,8 +168,15 @@ def run_with_manual_setup():
     #     dimension=1024
     # )
 
-    destination = LocalDestination(
-        output_dir='./result'
+    # destination = LocalDestination(
+    #     output_dir='./result'
+    # )
+
+    destination = MilvusDestination(
+        db_path='https://in03-5388093d0db1707.serverless.ali-cn-hangzhou.cloud.zilliz.com.cn', # zilliz连接地址
+        collection_name='textin_test_2', # 数据库collection名称
+        dimension=1024,  # 向量维度，需与 embed API 返回一致
+        api_key='872c3f5b3f3995c80dcda5c3d34f1f608815aef7671b6ee391ab37e40e79c892ce56d9c8c6565a03a3fd66da7e11b67f384c5c46'  # Zilliz Cloud API Key
     )
 
     # destination = S3Destination(
@@ -185,7 +192,7 @@ def run_with_manual_setup():
     stages = [
         Stage(
             type='parse',
-            config=ParseConfig(provider='textin')
+            config=ParseConfig(provider='textin-lite')
         ),
         Stage(
             type='chunk',
@@ -196,6 +203,13 @@ def run_with_manual_setup():
                 max_characters=1024,
                 overlap=50                     # 块之间重叠 50 字符
             )
+        ),
+        Stage(
+            type='embed',
+            config=EmbedConfig(
+                provider='qwen',
+                model_name='text-embedding-v3'
+            )
         )
     ]
     

xparse_client/pipeline/config.py

@@ -9,7 +9,7 @@ from .destinations import Destination
 class ParseConfig:
     """Parse 配置，支持动态字段"""
 
-    def __init__(self, provider: Literal["textin", "mineru", "paddle"] = "textin", **kwargs):
+    def __init__(self, provider: Literal["textin", "mineru", "paddle", "textin-lite"] = "textin", **kwargs):
         self.provider = provider
         for key, value in kwargs.items():
             setattr(self, key, value)

xparse_client/pipeline/destinations.py

@@ -18,6 +18,41 @@ from pymilvus import MilvusClient
 logger = logging.getLogger(__name__)
 
 
+def _flatten_dict(data: Dict[str, Any], prefix: str = '', fixed_fields: set = None) -> Dict[str, Any]:
+    """递归展平嵌套字典
+    
+    Args:
+        data: 要展平的字典
+        prefix: 键的前缀
+        fixed_fields: 需要排除的字段集合
+        
+    Returns:
+        展平后的字典
+    """
+    if fixed_fields is None:
+        fixed_fields = set()
+    
+    result = {}
+    for key, value in data.items():
+        flat_key = f'{prefix}_{key}' if prefix else key
+        
+        if flat_key in fixed_fields:
+            continue
+        
+        if isinstance(value, dict):
+            # 递归展平嵌套字典
+            nested = _flatten_dict(value, flat_key, fixed_fields)
+            result.update(nested)
+        elif isinstance(value, list):
+            # 列表转换为 JSON 字符串
+            result[flat_key] = json.dumps(value, ensure_ascii=False)
+        else:
+            # 其他类型直接使用
+            result[flat_key] = value
+    
+    return result
+
+
 class Destination(ABC):
     """数据目的地抽象基类"""
 
@@ -79,17 +114,42 @@ class MilvusDestination(Destination):
         try:
             insert_data = []
             for item in data:
-                metadata = item.get('metadata', {})
+                # 获取元素级别的 metadata
+                element_metadata = item.get('metadata', {})
+                
                 if 'embeddings' in item and item['embeddings']:
                     element_id = item.get('element_id') or item.get('id') or str(uuid.uuid4())
-                    insert_data.append({
+                    
+                    # 构建基础数据
+                    insert_item = {
                         'embeddings': item['embeddings'],
                         'text': item.get('text', ''),
                         'element_id': element_id,
-                        'record_id': metadata.get('record_id', ''),
-                        'metadata': metadata,
-                        'created_at': datetime.now().isoformat()
-                    })
+                        'record_id': element_metadata.get('record_id', '')
+                    }
+                    
+                    # 合并文件级别的 metadata 和元素级别的 metadata
+                    # 文件级别的 metadata 优先级更高
+                    merged_metadata = {**element_metadata, **metadata}
+                    
+                    # 将 metadata 中的字段展平到顶层作为动态字段
+                    # 排除已存在的固定字段，避免冲突
+                    fixed_fields = {'embeddings', 'text', 'element_id', 'record_id', 'created_at', 'metadata'}
+                    for key, value in merged_metadata.items():
+                        if key not in fixed_fields:
+                            # 特殊处理 data_source 字段：如果是字典则递归展平
+                            if key == 'data_source' and isinstance(value, dict):
+                                # 递归展平 data_source 字典，包括嵌套的字典
+                                flattened = _flatten_dict(value, 'data_source', fixed_fields)
+                                insert_item.update(flattened)
+                            elif key == 'coordinates' and isinstance(value, list):
+                                insert_item[key] = value
+                            elif isinstance(value, (dict, list)):
+                                continue
+                            else:
+                                insert_item[key] = value
+                    
+                    insert_data.append(insert_item)
 
             if not insert_data:
                 print(f"  ! 警告: 没有有效的向量数据")

xparse_client/pipeline/pipeline.py

@@ -198,7 +198,9 @@ class Pipeline:
             print(f"  → 读取文件...")
             file_bytes, data_source = self.source.read_file(file_path)
             data_source = data_source or {}
-            data_source['date_processed'] = datetime.now(timezone.utc).timestamp()
+            # 转换为毫秒时间戳字符串
+            timestamp_ms = int(datetime.now(timezone.utc).timestamp() * 1000)
+            data_source['date_processed'] = str(timestamp_ms)
             print(f"  ✓ 文件读取完成: {len(file_bytes)} bytes")
 
             result = self.process_with_pipeline(file_bytes, file_path, data_source)
@@ -209,15 +211,8 @@ class Pipeline:
 
             print(f"  → 写入目的地...")
             metadata = {
-                'file_name': file_path,
-                'total_elements': len(embedded_data),
-                'processed_at': datetime.now().isoformat(),
-                'data_source': data_source,
-                'stats': {
-                    'original_elements': stats.original_elements,
-                    'chunked_elements': stats.chunked_elements,
-                    'embedded_elements': stats.embedded_elements
-                }
+                'filename': file_path,
+                'processed_at': str(timestamp_ms),
             }
 
             success = self.destination.write(embedded_data, metadata)

xparse_client/pipeline/sources.py

@@ -20,6 +20,30 @@ from botocore.config import Config
 logger = logging.getLogger(__name__)
 
 
+def _to_millis_timestamp_string(timestamp):
+    """将时间戳转换为毫秒时间戳字符串
+    
+    Args:
+        timestamp: 时间戳（秒或毫秒），可以是 int、float 或 None
+        
+    Returns:
+        str: 毫秒时间戳字符串，如果输入为 None 则返回空字符串
+    """
+    if timestamp is None:
+        return ""
+    
+    # 如果已经是毫秒时间戳（大于 1e12），直接转换
+    if isinstance(timestamp, (int, float)):
+        if timestamp > 1e12:
+            # 已经是毫秒时间戳
+            return str(int(timestamp))
+        else:
+            # 秒级时间戳，转换为毫秒
+            return str(int(timestamp * 1000))
+    
+    return str(timestamp)
+
+
 class Source(ABC):
     """数据源抽象基类"""
 
@@ -106,8 +130,8 @@ class S3Source(Source):
         data_source = {
             'url': f"s3://{self.bucket}/{normalized_key}",
             'version': version,
-            'date_created': date_modified,
-            'date_modified': date_modified,
+            'date_created': _to_millis_timestamp_string(date_modified),
+            'date_modified': _to_millis_timestamp_string(date_modified),
             'record_locator': {
                 'server': server,
                 'protocol': 's3',
@@ -159,8 +183,8 @@ class LocalSource(Source):
         data_source = {
             'url': full_path.as_uri(),
             'version': version,
-            'date_created': date_created,
-            'date_modified': date_modified,
+            'date_created': _to_millis_timestamp_string(date_created),
+            'date_modified': _to_millis_timestamp_string(date_modified),
             'record_locator': {
                 'protocol': 'file',
                 'remote_file_path': str(full_path)
@@ -211,8 +235,8 @@ class FtpSource(Source):
         data_source = {
             'url': f"ftp://{self.host}:{self.port}/{normalized_path}",
             'version': None,
-            'date_created': None,
-            'date_modified': date_modified,
+            'date_created': "",
+            'date_modified': _to_millis_timestamp_string(date_modified),
             'record_locator': {
                 'server': f"{self.host}:{self.port}",
                 'protocol': 'ftp',
@@ -311,8 +335,8 @@ class SmbSource(Source):
         data_source = {
             'url': smb_url,
             'version': None,
-            'date_created': date_created,
-            'date_modified': date_modified,
+            'date_created': _to_millis_timestamp_string(date_created),
+            'date_modified': _to_millis_timestamp_string(date_modified),
             'record_locator': {
                 'server': self.host,
                 'share': self.share_name,

{xparse_client-0.2.3.dist-info → xparse_client-0.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xparse-client
-Version: 0.2.3
+Version: 0.2.5
 Summary: 面向Agent和RAG的新一代文档处理 AI Infra
 License-Expression: MIT
 Project-URL: Homepage, https://gitlab.intsig.net/xparse1/xparse-pipeline
@@ -331,9 +331,9 @@ source = SmbSource(
 
 ```python
 destination = MilvusDestination(
-    db_path: './milvus_pipeline.db', # 本地数据库文件
-    collection_name: 'my_collection', # 数据库collection名称
-    dimension: 1024  # 向量维度，需与 embed API 返回一致
+    db_path='./milvus_pipeline.db', # 本地数据库文件
+    collection_name='my_collection', # 数据库collection名称
+    dimension=1024  # 向量维度，需与 embed API 返回一致
 )
 ```
 
@@ -341,10 +341,10 @@ destination = MilvusDestination(
 
 ```python
 destination = MilvusDestination(
-    db_path: 'https://xxxxxxx.serverless.xxxxxxx.cloud.zilliz.com.cn', # zilliz连接地址
-    collection_name: 'my_collection', # 数据库collection名称
-    dimension: 1024,  # 向量维度，需与 embed API 返回一致
-    api_key: 'your-api-key'  # Zilliz Cloud API Key
+    db_path='https://xxxxxxx.serverless.xxxxxxx.cloud.zilliz.com.cn', # zilliz连接地址
+    collection_name='my_collection', # 数据库collection名称
+    dimension=1024,  # 向量维度，需与 embed API 返回一致
+    api_key='your-api-key'  # Zilliz Cloud API Key
 )
 ```
 
@@ -354,7 +354,7 @@ destination = MilvusDestination(
 
 ```python
 destination = LocalDestination(
-    output_dir: './output'
+    output_dir='./output'
 )
 ```
 

xparse_client-0.2.5.dist-info/RECORD

@@ -0,0 +1,13 @@
+example/run_pipeline.py,sha256=6gavTizAIqD62g4n9Pjq2-yW57ItZMJOOw8GEKm0Byk,15125
+example/run_pipeline_test.py,sha256=uIU09FTv_VnTQS1Lc94ydc3kaD86eHkaHQbVXpsGEcA,14861
+xparse_client/__init__.py,sha256=je1ena3HwLL4CRtLU4r6EAzoOIJthlPjTwshxZnzQDM,1677
+xparse_client/pipeline/__init__.py,sha256=TVlb2AGCNKP0jrv3p4ZLZCPKp68hTVMFi00DTdi6QAo,49
+xparse_client/pipeline/config.py,sha256=gkhAF-55PNvPPyfTZ0HkP95XB_K0HKCyYl6R4PTQLhI,4045
+xparse_client/pipeline/destinations.py,sha256=dXtsYw6xu_pBH8GKu9zKRzD6qP6Krgg-l1zCTzj8PhU,10475
+xparse_client/pipeline/pipeline.py,sha256=nWebrMGJjxTKF1de-7DeNtyozuUOkh3zbAqv-Q78Ev4,18039
+xparse_client/pipeline/sources.py,sha256=NdME4mTSzC7JXXhaw985Tt_nc0y_aRUl2NEALm8xLII,12484
+xparse_client-0.2.5.dist-info/licenses/LICENSE,sha256=ckIP-MbocsP9nqYnta5KgfAicYF196B5TNdHIR6kOO0,1075
+xparse_client-0.2.5.dist-info/METADATA,sha256=iN0qG5s4bc1hAueNx0VucDekicwdF-kQ5IzUWH2tJd4,26500
+xparse_client-0.2.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+xparse_client-0.2.5.dist-info/top_level.txt,sha256=bfX8BWo1sEEQVsI4Ql4Uu80vrfEh5zfajU9YqFTzxMo,22
+xparse_client-0.2.5.dist-info/RECORD,,

xparse_client-0.2.3.dist-info/RECORD

@@ -1,13 +0,0 @@
-example/run_pipeline.py,sha256=UNG18bBqTqFlkRp4liVtJFpXXnuiZQ6nuGO3LbeksCY,14424
-example/run_pipeline_test.py,sha256=uIU09FTv_VnTQS1Lc94ydc3kaD86eHkaHQbVXpsGEcA,14861
-xparse_client/__init__.py,sha256=je1ena3HwLL4CRtLU4r6EAzoOIJthlPjTwshxZnzQDM,1677
-xparse_client/pipeline/__init__.py,sha256=TVlb2AGCNKP0jrv3p4ZLZCPKp68hTVMFi00DTdi6QAo,49
-xparse_client/pipeline/config.py,sha256=n2T6Ptma7pAyfcbUcULsvIPNR__yXIIznojHxm_g0Rw,4030
-xparse_client/pipeline/destinations.py,sha256=JqTuSqRqJFpMdeDYlvevdF32Cti40QL1oR1pMv-uxuc,7976
-xparse_client/pipeline/pipeline.py,sha256=oz_BKWLbslkuRsxG0zEfh9url7saLWgtoTH1mrK6gCc,18282
-xparse_client/pipeline/sources.py,sha256=-0Eutg9t8xni12cfv2bdQVdImlkCQ7gWlOXIFBt6tpE,11568
-xparse_client-0.2.3.dist-info/licenses/LICENSE,sha256=ckIP-MbocsP9nqYnta5KgfAicYF196B5TNdHIR6kOO0,1075
-xparse_client-0.2.3.dist-info/METADATA,sha256=aumlSAr6sgrfppsMCQy_hUtkTJW4bZn70p6dhnO1nHw,26508
-xparse_client-0.2.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-xparse_client-0.2.3.dist-info/top_level.txt,sha256=bfX8BWo1sEEQVsI4Ql4Uu80vrfEh5zfajU9YqFTzxMo,22
-xparse_client-0.2.3.dist-info/RECORD,,