xparse-client 0.2.10__py3-none-any.whl → 0.2.12__py3-none-any.whl

example/run_pipeline.py

@@ -101,8 +101,15 @@ def run_with_manual_setup():
     #     secret_key='JFIIaTGiXelv7DgBYNIBSStofF0S98',
     #     bucket='textin',
     #     prefix='',
-    #     region='cn-shanghai',
-    #     pattern='*.png'
+    #     region='cn-shanghai'
+    # )
+    # source=S3Source(
+    #     endpoint='https://S3.oss-cn-shanghai.aliyuncs.com',
+    #     access_key='LTAI5t6ZnqTra8oLmJEfvcr7',
+    #     secret_key='SEbz4oJ4KNJIOTMfphuVGOWmRpGGUG',
+    #     bucket='textin-test-aliyun',
+    #     prefix='',
+    #     region='cn-shanghai'
     # )
     # source = S3Source(
     #     endpoint='https://cos.ap-shanghai.myqcloud.com',
@@ -128,14 +135,14 @@ def run_with_manual_setup():
     #     prefix='',
     #     region='cn-east-3'
     # )
-    source = S3Source(
-        endpoint='https://s3.us-east-1.amazonaws.com',
-        access_key='AKIA6QUE3TVZADUWA4PO',
-        secret_key='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',
-        bucket='textin-test',
-        prefix='',
-        region='us-east-1'
-    )
+    # source = S3Source(
+    #     endpoint='https://s3.us-east-1.amazonaws.com',
+    #     access_key='AKIA6QUE3TVZADUWA4PO',
+    #     secret_key='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',
+    #     bucket='textin-test',
+    #     prefix='',
+    #     region='us-east-1'
+    # )
     # source = S3Source(
     #     endpoint='http://127.0.0.1:9000',
     #     access_key='',
@@ -158,11 +165,20 @@ def run_with_manual_setup():
     #     username='', # 用户名，按照实际填写
     #     password=''  # 密码，按照实际填写
     # )
-    # source = LocalSource(
-    #     directory='/Users/ke_wang/Documents/doc',
-    #     recursive=True,
-    #     pattern=['**/*.png']  # 支持通配符: *.pdf, *.docx, **/*.txt
-    # )
+    source = LocalSource(
+        directory='/Users/ke_wang/Documents/doc',
+        pattern=['*.pdf'],
+        recursive=True,
+    )
+
+    # source=S3Source(
+    #     endpoint='https://obs.cn-north-4.myhuaweicloud.com',
+    #     access_key='HPUAFT3D1Q6O6UUN1RWQ',
+    #     secret_key='4zIk8x37nZiDS9P585BTFCWsOSo5G7ok1yRWtEA1',
+    #     bucket='textin-test-ywj',
+    #     prefix='',
+    #     region='cn-north-4'
+    # )# 华为云
     
     # 创建 Milvus 目的地
     # destination = MilvusDestination(
@@ -191,18 +207,27 @@ def run_with_manual_setup():
     #     region='ap-shanghai'
     # )
 
-    destination = QdrantDestination(
-        url='https://1325db22-7dd8-4fc9-930b-f969d4963b3d.us-east-1-1.aws.cloud.qdrant.io:6333',
-        collection_name='textin1',
-        dimension=1024,
-        api_key='eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3MiOiJtIn0.TGnFB1pAD7c7IqSOvTpgCPpHXSnnoKhWEQ5pQ8DrBnI',
-    )
+    # destination = QdrantDestination(
+    #     url='https://1325db22-7dd8-4fc9-930b-f969d4963b3d.us-east-1-1.aws.cloud.qdrant.io:6333',
+    #     collection_name='textin1',
+    #     dimension=1024,
+    #     api_key='eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3MiOiJtIn0.TGnFB1pAD7c7IqSOvTpgCPpHXSnnoKhWEQ5pQ8DrBnI',
+    # )
+
+    destination = S3Destination(
+            endpoint='https://obs.cn-east-3.myhuaweicloud.com',
+            access_key='HPUAFT3D1Q6O6UUN1RWQ',
+            secret_key='4zIk8x37nZiDS9P585BTFCWsOSo5G7ok1yRWtEA1',
+            bucket='xparse',
+            prefix='json/',
+            region='cn-east-3'
+        )  # 华为云
     
     # 使用新的 stages 格式创建配置
     stages = [
         Stage(
             type='parse',
-            config=ParseConfig(provider='paddle')
+            config=ParseConfig(provider='textin', page_ranges='3')
         ),
         Stage(
             type='chunk',

xparse_client/pipeline/destinations.py

@@ -240,6 +240,8 @@ class S3Destination(Destination):
             config = Config(signature_version='s3v4')
         elif self.endpoint.endswith('aliyuncs.com'):
             config = Config(signature_version='s3', s3={'addressing_style': 'virtual'})
+        elif self.endpoint.endswith('myhuaweicloud.com'):
+            config = Config(signature_version='s3', s3={'addressing_style': 'virtual'})
         else:
             config = Config(signature_version='s3v4', s3={'addressing_style': 'virtual'})
 

xparse_client/pipeline/sources.py

@@ -121,6 +121,8 @@ class S3Source(Source):
 
         if self.endpoint == 'https://textin-minio-api.ai.intsig.net':
             config = Config(signature_version='s3v4')
+        elif self.endpoint.endswith('aliyuncs.com'):
+            config = Config(signature_version='s3', s3={'addressing_style': 'virtual'})
         else:
             config = Config(signature_version='s3v4', s3={'addressing_style': 'virtual'})
 

{xparse_client-0.2.10.dist-info → xparse_client-0.2.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xparse-client
-Version: 0.2.10
+Version: 0.2.12
 Summary: 面向Agent和RAG的新一代文档处理 AI Infra
 License-Expression: MIT
 Project-URL: Homepage, https://gitlab.intsig.net/xparse1/xparse-pipeline
@@ -10,7 +10,8 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: boto3
-Requires-Dist: pymilvus[milvus_lite]
+Requires-Dist: pymilvus
+Requires-Dist: milvus-lite
 Requires-Dist: requests
 Requires-Dist: pysmb
 Requires-Dist: qdrant-client

xparse_client-0.2.12.dist-info/RECORD

@@ -0,0 +1,13 @@
+example/run_pipeline.py,sha256=AnNMUD0h6eoWdUQrejw6YZs7O_aM5rYajbtAAoAALrc,16671
+example/run_pipeline_test.py,sha256=pxsNiq_LmP6M4R7tTuja0u-Lu7fW-wIBU1uBf0-agQI,14845
+xparse_client/__init__.py,sha256=C2XLxkCoONl6_B1FmDhWRw84TqOL4pZF20br-K26SSY,1721
+xparse_client/pipeline/__init__.py,sha256=TVlb2AGCNKP0jrv3p4ZLZCPKp68hTVMFi00DTdi6QAo,49
+xparse_client/pipeline/config.py,sha256=FFYq2a0dBWBEj70s2aInXOiQ5MwwHimd6SI2_tkp52w,4138
+xparse_client/pipeline/destinations.py,sha256=QKlNGcpXIqkZS3rlBlhLDoRqIWA21Jgn3GiGhhfE8Rc,20921
+xparse_client/pipeline/pipeline.py,sha256=ZspagUjiL5wnzGJq6A7riOU8qGXJMtg1fqPm9H09mkk,27272
+xparse_client/pipeline/sources.py,sha256=pzJ5FjP-kZi-6Cphhm9rOPXETmHw5Qpf7EaxrQPgSxs,22285
+xparse_client-0.2.12.dist-info/licenses/LICENSE,sha256=ckIP-MbocsP9nqYnta5KgfAicYF196B5TNdHIR6kOO0,1075
+xparse_client-0.2.12.dist-info/METADATA,sha256=wQIOLJAjxSbLGpWXNM_rZmKK2h_TSoNwHvvhQLYafaw,28799
+xparse_client-0.2.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+xparse_client-0.2.12.dist-info/top_level.txt,sha256=bfX8BWo1sEEQVsI4Ql4Uu80vrfEh5zfajU9YqFTzxMo,22
+xparse_client-0.2.12.dist-info/RECORD,,

xparse_client-0.2.10.dist-info/RECORD

@@ -1,13 +0,0 @@
-example/run_pipeline.py,sha256=d4pPDqjiC9dPNh6nmArPOF7fPMY0a-jcvdgtNuV-_kM,15795
-example/run_pipeline_test.py,sha256=pxsNiq_LmP6M4R7tTuja0u-Lu7fW-wIBU1uBf0-agQI,14845
-xparse_client/__init__.py,sha256=C2XLxkCoONl6_B1FmDhWRw84TqOL4pZF20br-K26SSY,1721
-xparse_client/pipeline/__init__.py,sha256=TVlb2AGCNKP0jrv3p4ZLZCPKp68hTVMFi00DTdi6QAo,49
-xparse_client/pipeline/config.py,sha256=FFYq2a0dBWBEj70s2aInXOiQ5MwwHimd6SI2_tkp52w,4138
-xparse_client/pipeline/destinations.py,sha256=9UyZ8Ygjoe4yAq6-VZNZBoNYRbb3mahify3c1AdOHMY,20775
-xparse_client/pipeline/pipeline.py,sha256=ZspagUjiL5wnzGJq6A7riOU8qGXJMtg1fqPm9H09mkk,27272
-xparse_client/pipeline/sources.py,sha256=D-kLrSQ-qsFFFq7JC4sL3Y3Q3Q87Wcpv9R5K85YkDjE,22144
-xparse_client-0.2.10.dist-info/licenses/LICENSE,sha256=ckIP-MbocsP9nqYnta5KgfAicYF196B5TNdHIR6kOO0,1075
-xparse_client-0.2.10.dist-info/METADATA,sha256=gIY_PxB1pTxSlKJZjU7z1Iua6ZMtAfMfHFeztWp2zIw,28785
-xparse_client-0.2.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-xparse_client-0.2.10.dist-info/top_level.txt,sha256=bfX8BWo1sEEQVsI4Ql4Uu80vrfEh5zfajU9YqFTzxMo,22
-xparse_client-0.2.10.dist-info/RECORD,,