xparse-client 0.2.4__py3-none-any.whl → 0.2.5__py3-none-any.whl

xparse_client/pipeline/destinations.py

@@ -18,6 +18,41 @@ from pymilvus import MilvusClient
 logger = logging.getLogger(__name__)
 
 
+def _flatten_dict(data: Dict[str, Any], prefix: str = '', fixed_fields: set = None) -> Dict[str, Any]:
+    """递归展平嵌套字典
+    
+    Args:
+        data: 要展平的字典
+        prefix: 键的前缀
+        fixed_fields: 需要排除的字段集合
+        
+    Returns:
+        展平后的字典
+    """
+    if fixed_fields is None:
+        fixed_fields = set()
+    
+    result = {}
+    for key, value in data.items():
+        flat_key = f'{prefix}_{key}' if prefix else key
+        
+        if flat_key in fixed_fields:
+            continue
+        
+        if isinstance(value, dict):
+            # 递归展平嵌套字典
+            nested = _flatten_dict(value, flat_key, fixed_fields)
+            result.update(nested)
+        elif isinstance(value, list):
+            # 列表转换为 JSON 字符串
+            result[flat_key] = json.dumps(value, ensure_ascii=False)
+        else:
+            # 其他类型直接使用
+            result[flat_key] = value
+    
+    return result
+
+
 class Destination(ABC):
     """数据目的地抽象基类"""
 
@@ -90,8 +125,7 @@ class MilvusDestination(Destination):
                         'embeddings': item['embeddings'],
                         'text': item.get('text', ''),
                         'element_id': element_id,
-                        'record_id': element_metadata.get('record_id', ''),
-                        'created_at': datetime.now().isoformat()
+                        'record_id': element_metadata.get('record_id', '')
                     }
                     
                     # 合并文件级别的 metadata 和元素级别的 metadata
@@ -103,17 +137,13 @@ class MilvusDestination(Destination):
                     fixed_fields = {'embeddings', 'text', 'element_id', 'record_id', 'created_at', 'metadata'}
                     for key, value in merged_metadata.items():
                         if key not in fixed_fields:
-                            # 特殊处理 data_source 字段：如果是字典则展平
+                            # 特殊处理 data_source 字段：如果是字典则递归展平
                             if key == 'data_source' and isinstance(value, dict):
-                                # 将 data_source 字典展平为 data_source_* 格式
-                                for sub_key, sub_value in value.items():
-                                    flat_key = f'data_source_{sub_key}'
-                                    if flat_key not in fixed_fields:
-                                        # 如果子值也是字典或列表，转换为 JSON 字符串
-                                        if isinstance(sub_value, (dict, list)):
-                                            insert_item[flat_key] = json.dumps(sub_value, ensure_ascii=False)
-                                        else:
-                                            insert_item[flat_key] = sub_value
+                                # 递归展平 data_source 字典，包括嵌套的字典
+                                flattened = _flatten_dict(value, 'data_source', fixed_fields)
+                                insert_item.update(flattened)
+                            elif key == 'coordinates' and isinstance(value, list):
+                                insert_item[key] = value
                             elif isinstance(value, (dict, list)):
                                 continue
                             else:

xparse_client/pipeline/pipeline.py

@@ -198,7 +198,9 @@ class Pipeline:
             print(f"  → 读取文件...")
             file_bytes, data_source = self.source.read_file(file_path)
             data_source = data_source or {}
-            data_source['date_processed'] = datetime.now(timezone.utc).timestamp()
+            # 转换为毫秒时间戳字符串
+            timestamp_ms = int(datetime.now(timezone.utc).timestamp() * 1000)
+            data_source['date_processed'] = str(timestamp_ms)
             print(f"  ✓ 文件读取完成: {len(file_bytes)} bytes")
 
             result = self.process_with_pipeline(file_bytes, file_path, data_source)
@@ -209,15 +211,8 @@ class Pipeline:
 
             print(f"  → 写入目的地...")
             metadata = {
-                'file_name': file_path,
-                'total_elements': len(embedded_data),
-                'processed_at': datetime.now().isoformat(),
-                'data_source': data_source,
-                'stats': {
-                    'original_elements': stats.original_elements,
-                    'chunked_elements': stats.chunked_elements,
-                    'embedded_elements': stats.embedded_elements
-                }
+                'filename': file_path,
+                'processed_at': str(timestamp_ms),
             }
 
             success = self.destination.write(embedded_data, metadata)

xparse_client/pipeline/sources.py

@@ -20,6 +20,30 @@ from botocore.config import Config
 logger = logging.getLogger(__name__)
 
 
+def _to_millis_timestamp_string(timestamp):
+    """将时间戳转换为毫秒时间戳字符串
+    
+    Args:
+        timestamp: 时间戳（秒或毫秒），可以是 int、float 或 None
+        
+    Returns:
+        str: 毫秒时间戳字符串，如果输入为 None 则返回空字符串
+    """
+    if timestamp is None:
+        return ""
+    
+    # 如果已经是毫秒时间戳（大于 1e12），直接转换
+    if isinstance(timestamp, (int, float)):
+        if timestamp > 1e12:
+            # 已经是毫秒时间戳
+            return str(int(timestamp))
+        else:
+            # 秒级时间戳，转换为毫秒
+            return str(int(timestamp * 1000))
+    
+    return str(timestamp)
+
+
 class Source(ABC):
     """数据源抽象基类"""
 
@@ -106,8 +130,8 @@ class S3Source(Source):
         data_source = {
             'url': f"s3://{self.bucket}/{normalized_key}",
             'version': version,
-            'date_created': date_modified,
-            'date_modified': date_modified,
+            'date_created': _to_millis_timestamp_string(date_modified),
+            'date_modified': _to_millis_timestamp_string(date_modified),
             'record_locator': {
                 'server': server,
                 'protocol': 's3',
@@ -159,8 +183,8 @@ class LocalSource(Source):
         data_source = {
             'url': full_path.as_uri(),
             'version': version,
-            'date_created': date_created,
-            'date_modified': date_modified,
+            'date_created': _to_millis_timestamp_string(date_created),
+            'date_modified': _to_millis_timestamp_string(date_modified),
             'record_locator': {
                 'protocol': 'file',
                 'remote_file_path': str(full_path)
@@ -211,8 +235,8 @@ class FtpSource(Source):
         data_source = {
             'url': f"ftp://{self.host}:{self.port}/{normalized_path}",
             'version': None,
-            'date_created': None,
-            'date_modified': date_modified,
+            'date_created': "",
+            'date_modified': _to_millis_timestamp_string(date_modified),
             'record_locator': {
                 'server': f"{self.host}:{self.port}",
                 'protocol': 'ftp',
@@ -311,8 +335,8 @@ class SmbSource(Source):
         data_source = {
             'url': smb_url,
             'version': None,
-            'date_created': date_created,
-            'date_modified': date_modified,
+            'date_created': _to_millis_timestamp_string(date_created),
+            'date_modified': _to_millis_timestamp_string(date_modified),
             'record_locator': {
                 'server': self.host,
                 'share': self.share_name,

{xparse_client-0.2.4.dist-info → xparse_client-0.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xparse-client
-Version: 0.2.4
+Version: 0.2.5
 Summary: 面向Agent和RAG的新一代文档处理 AI Infra
 License-Expression: MIT
 Project-URL: Homepage, https://gitlab.intsig.net/xparse1/xparse-pipeline
@@ -331,9 +331,9 @@ source = SmbSource(
 
 ```python
 destination = MilvusDestination(
-    db_path: './milvus_pipeline.db', # 本地数据库文件
-    collection_name: 'my_collection', # 数据库collection名称
-    dimension: 1024  # 向量维度，需与 embed API 返回一致
+    db_path='./milvus_pipeline.db', # 本地数据库文件
+    collection_name='my_collection', # 数据库collection名称
+    dimension=1024  # 向量维度，需与 embed API 返回一致
 )
 ```
 
@@ -341,10 +341,10 @@ destination = MilvusDestination(
 
 ```python
 destination = MilvusDestination(
-    db_path: 'https://xxxxxxx.serverless.xxxxxxx.cloud.zilliz.com.cn', # zilliz连接地址
-    collection_name: 'my_collection', # 数据库collection名称
-    dimension: 1024,  # 向量维度，需与 embed API 返回一致
-    api_key: 'your-api-key'  # Zilliz Cloud API Key
+    db_path='https://xxxxxxx.serverless.xxxxxxx.cloud.zilliz.com.cn', # zilliz连接地址
+    collection_name='my_collection', # 数据库collection名称
+    dimension=1024,  # 向量维度，需与 embed API 返回一致
+    api_key='your-api-key'  # Zilliz Cloud API Key
 )
 ```
 
@@ -354,7 +354,7 @@ destination = MilvusDestination(
 
 ```python
 destination = LocalDestination(
-    output_dir: './output'
+    output_dir='./output'
 )
 ```
 

xparse_client-0.2.5.dist-info/RECORD

@@ -0,0 +1,13 @@
+example/run_pipeline.py,sha256=6gavTizAIqD62g4n9Pjq2-yW57ItZMJOOw8GEKm0Byk,15125
+example/run_pipeline_test.py,sha256=uIU09FTv_VnTQS1Lc94ydc3kaD86eHkaHQbVXpsGEcA,14861
+xparse_client/__init__.py,sha256=je1ena3HwLL4CRtLU4r6EAzoOIJthlPjTwshxZnzQDM,1677
+xparse_client/pipeline/__init__.py,sha256=TVlb2AGCNKP0jrv3p4ZLZCPKp68hTVMFi00DTdi6QAo,49
+xparse_client/pipeline/config.py,sha256=gkhAF-55PNvPPyfTZ0HkP95XB_K0HKCyYl6R4PTQLhI,4045
+xparse_client/pipeline/destinations.py,sha256=dXtsYw6xu_pBH8GKu9zKRzD6qP6Krgg-l1zCTzj8PhU,10475
+xparse_client/pipeline/pipeline.py,sha256=nWebrMGJjxTKF1de-7DeNtyozuUOkh3zbAqv-Q78Ev4,18039
+xparse_client/pipeline/sources.py,sha256=NdME4mTSzC7JXXhaw985Tt_nc0y_aRUl2NEALm8xLII,12484
+xparse_client-0.2.5.dist-info/licenses/LICENSE,sha256=ckIP-MbocsP9nqYnta5KgfAicYF196B5TNdHIR6kOO0,1075
+xparse_client-0.2.5.dist-info/METADATA,sha256=iN0qG5s4bc1hAueNx0VucDekicwdF-kQ5IzUWH2tJd4,26500
+xparse_client-0.2.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+xparse_client-0.2.5.dist-info/top_level.txt,sha256=bfX8BWo1sEEQVsI4Ql4Uu80vrfEh5zfajU9YqFTzxMo,22
+xparse_client-0.2.5.dist-info/RECORD,,

xparse_client-0.2.4.dist-info/RECORD

@@ -1,13 +0,0 @@
-example/run_pipeline.py,sha256=6gavTizAIqD62g4n9Pjq2-yW57ItZMJOOw8GEKm0Byk,15125
-example/run_pipeline_test.py,sha256=uIU09FTv_VnTQS1Lc94ydc3kaD86eHkaHQbVXpsGEcA,14861
-xparse_client/__init__.py,sha256=je1ena3HwLL4CRtLU4r6EAzoOIJthlPjTwshxZnzQDM,1677
-xparse_client/pipeline/__init__.py,sha256=TVlb2AGCNKP0jrv3p4ZLZCPKp68hTVMFi00DTdi6QAo,49
-xparse_client/pipeline/config.py,sha256=gkhAF-55PNvPPyfTZ0HkP95XB_K0HKCyYl6R4PTQLhI,4045
-xparse_client/pipeline/destinations.py,sha256=rqcxmsn1YGClVxGQxSVmyr-uumOVilOv_vX82fUBj-I,9859
-xparse_client/pipeline/pipeline.py,sha256=oz_BKWLbslkuRsxG0zEfh9url7saLWgtoTH1mrK6gCc,18282
-xparse_client/pipeline/sources.py,sha256=-0Eutg9t8xni12cfv2bdQVdImlkCQ7gWlOXIFBt6tpE,11568
-xparse_client-0.2.4.dist-info/licenses/LICENSE,sha256=ckIP-MbocsP9nqYnta5KgfAicYF196B5TNdHIR6kOO0,1075
-xparse_client-0.2.4.dist-info/METADATA,sha256=IMgXO9a7wnN0Ygzauk7eOkyrRFE3A2rq73eofvq3wBs,26508
-xparse_client-0.2.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-xparse_client-0.2.4.dist-info/top_level.txt,sha256=bfX8BWo1sEEQVsI4Ql4Uu80vrfEh5zfajU9YqFTzxMo,22
-xparse_client-0.2.4.dist-info/RECORD,,