xparse-client 0.2.3__py3-none-any.whl → 0.2.4__py3-none-any.whl

example/run_pipeline.py

@@ -15,8 +15,8 @@ from xparse_client import create_pipeline_from_config, S3Source, LocalSource, Mi
 
 # API 请求头配置
 API_HEADERS = {
-    'x-ti-app-id': '',
-    'x-ti-secret-code': ''
+    'x-ti-app-id': '4c0032d9e4d93b0ad674cac0d75256e7',
+    'x-ti-secret-code': '7104f599ad02b8468fc619f7605d2d8d'
 }
 
 
@@ -168,8 +168,15 @@ def run_with_manual_setup():
     #     dimension=1024
     # )
 
-    destination = LocalDestination(
-        output_dir='./result'
+    # destination = LocalDestination(
+    #     output_dir='./result'
+    # )
+
+    destination = MilvusDestination(
+        db_path='https://in03-5388093d0db1707.serverless.ali-cn-hangzhou.cloud.zilliz.com.cn', # zilliz连接地址
+        collection_name='textin_test_2', # 数据库collection名称
+        dimension=1024,  # 向量维度，需与 embed API 返回一致
+        api_key='872c3f5b3f3995c80dcda5c3d34f1f608815aef7671b6ee391ab37e40e79c892ce56d9c8c6565a03a3fd66da7e11b67f384c5c46'  # Zilliz Cloud API Key
     )
 
     # destination = S3Destination(
@@ -185,7 +192,7 @@ def run_with_manual_setup():
     stages = [
         Stage(
             type='parse',
-            config=ParseConfig(provider='textin')
+            config=ParseConfig(provider='textin-lite')
         ),
         Stage(
             type='chunk',
@@ -196,6 +203,13 @@ def run_with_manual_setup():
                 max_characters=1024,
                 overlap=50                     # 块之间重叠 50 字符
             )
+        ),
+        Stage(
+            type='embed',
+            config=EmbedConfig(
+                provider='qwen',
+                model_name='text-embedding-v3'
+            )
         )
     ]
     

xparse_client/pipeline/config.py

@@ -9,7 +9,7 @@ from .destinations import Destination
 class ParseConfig:
     """Parse 配置，支持动态字段"""
 
-    def __init__(self, provider: Literal["textin", "mineru", "paddle"] = "textin", **kwargs):
+    def __init__(self, provider: Literal["textin", "mineru", "paddle", "textin-lite"] = "textin", **kwargs):
         self.provider = provider
         for key, value in kwargs.items():
             setattr(self, key, value)

xparse_client/pipeline/destinations.py

@@ -79,17 +79,47 @@ class MilvusDestination(Destination):
         try:
             insert_data = []
             for item in data:
-                metadata = item.get('metadata', {})
+                # 获取元素级别的 metadata
+                element_metadata = item.get('metadata', {})
+                
                 if 'embeddings' in item and item['embeddings']:
                     element_id = item.get('element_id') or item.get('id') or str(uuid.uuid4())
-                    insert_data.append({
+                    
+                    # 构建基础数据
+                    insert_item = {
                         'embeddings': item['embeddings'],
                         'text': item.get('text', ''),
                         'element_id': element_id,
-                        'record_id': metadata.get('record_id', ''),
-                        'metadata': metadata,
+                        'record_id': element_metadata.get('record_id', ''),
                         'created_at': datetime.now().isoformat()
-                    })
+                    }
+                    
+                    # 合并文件级别的 metadata 和元素级别的 metadata
+                    # 文件级别的 metadata 优先级更高
+                    merged_metadata = {**element_metadata, **metadata}
+                    
+                    # 将 metadata 中的字段展平到顶层作为动态字段
+                    # 排除已存在的固定字段，避免冲突
+                    fixed_fields = {'embeddings', 'text', 'element_id', 'record_id', 'created_at', 'metadata'}
+                    for key, value in merged_metadata.items():
+                        if key not in fixed_fields:
+                            # 特殊处理 data_source 字段：如果是字典则展平
+                            if key == 'data_source' and isinstance(value, dict):
+                                # 将 data_source 字典展平为 data_source_* 格式
+                                for sub_key, sub_value in value.items():
+                                    flat_key = f'data_source_{sub_key}'
+                                    if flat_key not in fixed_fields:
+                                        # 如果子值也是字典或列表，转换为 JSON 字符串
+                                        if isinstance(sub_value, (dict, list)):
+                                            insert_item[flat_key] = json.dumps(sub_value, ensure_ascii=False)
+                                        else:
+                                            insert_item[flat_key] = sub_value
+                            elif isinstance(value, (dict, list)):
+                                continue
+                            else:
+                                insert_item[key] = value
+                    
+                    insert_data.append(insert_item)
 
             if not insert_data:
                 print(f"  ! 警告: 没有有效的向量数据")

{xparse_client-0.2.3.dist-info → xparse_client-0.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xparse-client
-Version: 0.2.3
+Version: 0.2.4
 Summary: 面向Agent和RAG的新一代文档处理 AI Infra
 License-Expression: MIT
 Project-URL: Homepage, https://gitlab.intsig.net/xparse1/xparse-pipeline

xparse_client-0.2.4.dist-info/RECORD

@@ -0,0 +1,13 @@
+example/run_pipeline.py,sha256=6gavTizAIqD62g4n9Pjq2-yW57ItZMJOOw8GEKm0Byk,15125
+example/run_pipeline_test.py,sha256=uIU09FTv_VnTQS1Lc94ydc3kaD86eHkaHQbVXpsGEcA,14861
+xparse_client/__init__.py,sha256=je1ena3HwLL4CRtLU4r6EAzoOIJthlPjTwshxZnzQDM,1677
+xparse_client/pipeline/__init__.py,sha256=TVlb2AGCNKP0jrv3p4ZLZCPKp68hTVMFi00DTdi6QAo,49
+xparse_client/pipeline/config.py,sha256=gkhAF-55PNvPPyfTZ0HkP95XB_K0HKCyYl6R4PTQLhI,4045
+xparse_client/pipeline/destinations.py,sha256=rqcxmsn1YGClVxGQxSVmyr-uumOVilOv_vX82fUBj-I,9859
+xparse_client/pipeline/pipeline.py,sha256=oz_BKWLbslkuRsxG0zEfh9url7saLWgtoTH1mrK6gCc,18282
+xparse_client/pipeline/sources.py,sha256=-0Eutg9t8xni12cfv2bdQVdImlkCQ7gWlOXIFBt6tpE,11568
+xparse_client-0.2.4.dist-info/licenses/LICENSE,sha256=ckIP-MbocsP9nqYnta5KgfAicYF196B5TNdHIR6kOO0,1075
+xparse_client-0.2.4.dist-info/METADATA,sha256=IMgXO9a7wnN0Ygzauk7eOkyrRFE3A2rq73eofvq3wBs,26508
+xparse_client-0.2.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+xparse_client-0.2.4.dist-info/top_level.txt,sha256=bfX8BWo1sEEQVsI4Ql4Uu80vrfEh5zfajU9YqFTzxMo,22
+xparse_client-0.2.4.dist-info/RECORD,,

xparse_client-0.2.3.dist-info/RECORD

@@ -1,13 +0,0 @@
-example/run_pipeline.py,sha256=UNG18bBqTqFlkRp4liVtJFpXXnuiZQ6nuGO3LbeksCY,14424
-example/run_pipeline_test.py,sha256=uIU09FTv_VnTQS1Lc94ydc3kaD86eHkaHQbVXpsGEcA,14861
-xparse_client/__init__.py,sha256=je1ena3HwLL4CRtLU4r6EAzoOIJthlPjTwshxZnzQDM,1677
-xparse_client/pipeline/__init__.py,sha256=TVlb2AGCNKP0jrv3p4ZLZCPKp68hTVMFi00DTdi6QAo,49
-xparse_client/pipeline/config.py,sha256=n2T6Ptma7pAyfcbUcULsvIPNR__yXIIznojHxm_g0Rw,4030
-xparse_client/pipeline/destinations.py,sha256=JqTuSqRqJFpMdeDYlvevdF32Cti40QL1oR1pMv-uxuc,7976
-xparse_client/pipeline/pipeline.py,sha256=oz_BKWLbslkuRsxG0zEfh9url7saLWgtoTH1mrK6gCc,18282
-xparse_client/pipeline/sources.py,sha256=-0Eutg9t8xni12cfv2bdQVdImlkCQ7gWlOXIFBt6tpE,11568
-xparse_client-0.2.3.dist-info/licenses/LICENSE,sha256=ckIP-MbocsP9nqYnta5KgfAicYF196B5TNdHIR6kOO0,1075
-xparse_client-0.2.3.dist-info/METADATA,sha256=aumlSAr6sgrfppsMCQy_hUtkTJW4bZn70p6dhnO1nHw,26508
-xparse_client-0.2.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-xparse_client-0.2.3.dist-info/top_level.txt,sha256=bfX8BWo1sEEQVsI4Ql4Uu80vrfEh5zfajU9YqFTzxMo,22
-xparse_client-0.2.3.dist-info/RECORD,,