xparse-client 0.2.3__tar.gz → 0.2.4__tar.gz

{xparse_client-0.2.3 → xparse_client-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xparse-client
-Version: 0.2.3
+Version: 0.2.4
 Summary: 面向Agent和RAG的新一代文档处理 AI Infra
 License-Expression: MIT
 Project-URL: Homepage, https://gitlab.intsig.net/xparse1/xparse-pipeline

{xparse_client-0.2.3 → xparse_client-0.2.4}/example/run_pipeline.py

@@ -15,8 +15,8 @@ from xparse_client import create_pipeline_from_config, S3Source, LocalSource, Mi
 
 # API 请求头配置
 API_HEADERS = {
-    'x-ti-app-id': '',
-    'x-ti-secret-code': ''
+    'x-ti-app-id': '4c0032d9e4d93b0ad674cac0d75256e7',
+    'x-ti-secret-code': '7104f599ad02b8468fc619f7605d2d8d'
 }
 
 
@@ -168,8 +168,15 @@ def run_with_manual_setup():
     #     dimension=1024
     # )
 
-    destination = LocalDestination(
-        output_dir='./result'
+    # destination = LocalDestination(
+    #     output_dir='./result'
+    # )
+
+    destination = MilvusDestination(
+        db_path='https://in03-5388093d0db1707.serverless.ali-cn-hangzhou.cloud.zilliz.com.cn', # zilliz连接地址
+        collection_name='textin_test_2', # 数据库collection名称
+        dimension=1024,  # 向量维度，需与 embed API 返回一致
+        api_key='872c3f5b3f3995c80dcda5c3d34f1f608815aef7671b6ee391ab37e40e79c892ce56d9c8c6565a03a3fd66da7e11b67f384c5c46'  # Zilliz Cloud API Key
     )
 
     # destination = S3Destination(
@@ -185,7 +192,7 @@ def run_with_manual_setup():
     stages = [
         Stage(
             type='parse',
-            config=ParseConfig(provider='textin')
+            config=ParseConfig(provider='textin-lite')
         ),
         Stage(
             type='chunk',
@@ -196,6 +203,13 @@ def run_with_manual_setup():
                 max_characters=1024,
                 overlap=50                     # 块之间重叠 50 字符
             )
+        ),
+        Stage(
+            type='embed',
+            config=EmbedConfig(
+                provider='qwen',
+                model_name='text-embedding-v3'
+            )
         )
     ]
     

{xparse_client-0.2.3 → xparse_client-0.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "xparse-client"
-version = "0.2.3"
+version = "0.2.4"
 description = "面向Agent和RAG的新一代文档处理 AI Infra"
 readme = "README.md"
 license = "MIT"

{xparse_client-0.2.3 → xparse_client-0.2.4}/xparse_client/pipeline/config.py

@@ -9,7 +9,7 @@ from .destinations import Destination
 class ParseConfig:
     """Parse 配置，支持动态字段"""
 
-    def __init__(self, provider: Literal["textin", "mineru", "paddle"] = "textin", **kwargs):
+    def __init__(self, provider: Literal["textin", "mineru", "paddle", "textin-lite"] = "textin", **kwargs):
         self.provider = provider
         for key, value in kwargs.items():
             setattr(self, key, value)

{xparse_client-0.2.3 → xparse_client-0.2.4}/xparse_client/pipeline/destinations.py

@@ -79,17 +79,47 @@ class MilvusDestination(Destination):
         try:
             insert_data = []
             for item in data:
-                metadata = item.get('metadata', {})
+                # 获取元素级别的 metadata
+                element_metadata = item.get('metadata', {})
+                
                 if 'embeddings' in item and item['embeddings']:
                     element_id = item.get('element_id') or item.get('id') or str(uuid.uuid4())
-                    insert_data.append({
+                    
+                    # 构建基础数据
+                    insert_item = {
                         'embeddings': item['embeddings'],
                         'text': item.get('text', ''),
                         'element_id': element_id,
-                        'record_id': metadata.get('record_id', ''),
-                        'metadata': metadata,
+                        'record_id': element_metadata.get('record_id', ''),
                         'created_at': datetime.now().isoformat()
-                    })
+                    }
+                    
+                    # 合并文件级别的 metadata 和元素级别的 metadata
+                    # 文件级别的 metadata 优先级更高
+                    merged_metadata = {**element_metadata, **metadata}
+                    
+                    # 将 metadata 中的字段展平到顶层作为动态字段
+                    # 排除已存在的固定字段，避免冲突
+                    fixed_fields = {'embeddings', 'text', 'element_id', 'record_id', 'created_at', 'metadata'}
+                    for key, value in merged_metadata.items():
+                        if key not in fixed_fields:
+                            # 特殊处理 data_source 字段：如果是字典则展平
+                            if key == 'data_source' and isinstance(value, dict):
+                                # 将 data_source 字典展平为 data_source_* 格式
+                                for sub_key, sub_value in value.items():
+                                    flat_key = f'data_source_{sub_key}'
+                                    if flat_key not in fixed_fields:
+                                        # 如果子值也是字典或列表，转换为 JSON 字符串
+                                        if isinstance(sub_value, (dict, list)):
+                                            insert_item[flat_key] = json.dumps(sub_value, ensure_ascii=False)
+                                        else:
+                                            insert_item[flat_key] = sub_value
+                            elif isinstance(value, (dict, list)):
+                                continue
+                            else:
+                                insert_item[key] = value
+                    
+                    insert_data.append(insert_item)
 
             if not insert_data:
                 print(f"  ! 警告: 没有有效的向量数据")

{xparse_client-0.2.3 → xparse_client-0.2.4}/xparse_client.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xparse-client
-Version: 0.2.3
+Version: 0.2.4
 Summary: 面向Agent和RAG的新一代文档处理 AI Infra
 License-Expression: MIT
 Project-URL: Homepage, https://gitlab.intsig.net/xparse1/xparse-pipeline