deepfos 1.1.63__tar.gz → 1.1.64__tar.gz

{deepfos-1.1.63 → deepfos-1.1.64}/CHANGELOG.md

@@ -1,3 +1,11 @@
+## [1.1.64] - 2025-07-08
+
+### 更新
+
+* 更新连接器元素获取连接信息的接口和密码处理逻辑
+* TaskUtil的api实例化延迟至TaskUtil实例化过程中
+
+
 ## [1.1.63] - 2025-07-01
 
 ### 更新

{deepfos-1.1.63 → deepfos-1.1.64}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.63
+Version: 1.1.64
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team

{deepfos-1.1.63 → deepfos-1.1.64}/deepfos/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-07-01T03:16:02+0000",
+ "date": "2025-07-08T07:07:33+0000",
  "dirty": false,
  "error": null,
- "full-revisionid": "ee27ae20cda4c8955b87eabb5b3b1c73e528e8e3",
- "version": "1.1.63"
+ "full-revisionid": "53d949a38fa538718df6900080ebd8b0bc1358c9",
+ "version": "1.1.64"
 }
 '''  # END VERSION_JSON
 

{deepfos-1.1.63 → deepfos-1.1.64}/deepfos/api/deepconnector.py

@@ -3,7 +3,7 @@ from typing import Union, Awaitable
 from deepfos.element.base import T_ElementInfoWithServer
 from deepfos.lib.decorator import cached_property
 
-from .base import ChildAPI, get, RootAPI
+from .base import ChildAPI, RootAPI, post
 from .models.deepconnector import *
 
 
@@ -14,10 +14,10 @@ class DataSourceAPI(ChildAPI):
     """连接信息相关接口"""
     endpoint = '/apis/v3/ds/spaces/{space}/apps/{app}'
 
-    @get('connection-info', data_wrapped=False)
+    @post('connection-info/query', data_wrapped=False)
     def connection_info(self, element_info: T_ElementInfoWithServer) -> Union[ConnectionInfoVo, Awaitable[ConnectionInfoVo]]:
         return {
-            'param': {
+            'body': {
                 'elementName': element_info.elementName,
                 'folderId': element_info.folderId
             }

{deepfos-1.1.63 → deepfos-1.1.64}/deepfos/api/models/deepconnector.py

@@ -26,3 +26,4 @@ class ConnectionInfoVo(BaseModel):
     serviceTypeName: Optional[str] = None
     serviceVersion: Optional[str] = None
     username: str
+    encryptType: Optional[str] = None

{deepfos-1.1.63 → deepfos-1.1.64}/deepfos/element/deepconnector.py

@@ -1,7 +1,10 @@
 import base64
+import binascii
+import os
 
 from deepfos.api.deepconnector import DeepConnectorAPI
 from deepfos.api.models import BaseModel
+from deepfos.db.cipher import AES
 from deepfos.element.base import ElementBase, SyncMeta
 from deepfos.lib.asynchronous import future_property
 from deepfos.lib.decorator import cached_property
@@ -9,6 +12,20 @@ from deepfos.lib.decorator import cached_property
 __all__ = ['AsyncDeepConnector', 'DeepConnector', 'ConnectionInfo']
 
 
+def decrypt(secret, cipher_text, encoding='utf8'):
+    pwd_padded = AES(secret).decrypt(
+        base64.b16decode(cipher_text)
+    )
+    if pwd_padded[-1] in range(1, 17):
+        pad_length = pwd_padded[-1]
+        pad_char = chr(pwd_padded[-1])
+        guess_pad = pad_char * pwd_padded[-1]
+
+        if pwd_padded.decode(encoding).endswith(guess_pad):
+            return pwd_padded.decode(encoding)[:-pad_length:]
+    return pwd_padded.decode(encoding)
+
+
 class ConnectionInfo(BaseModel):
     host: str
     port: int
@@ -45,12 +62,28 @@ class AsyncDeepConnector(ElementBase[DeepConnectorAPI]):
         info = await api.datasource.connection_info(
             element_info=ele_info,
         )
+        if info.encryptType == 'AES':
+            try:
+                password = decrypt(
+                    os.environ.get('EXPORT_DEEPFOS_AES_KEY', '!ABCD-EFGH-IJKL@').encode(),
+                    info.password,
+                    encoding='utf-8'
+                )
+            except ValueError:
+                raise ValueError(
+                    '连接器连接信息解密失败，请检查公共环境变量：EXPORT_DEEPFOS_AES_KEY'
+                ) from None
+        else:
+            try:
+                password = base64.decodebytes(info.password.encode()).decode()
+            except binascii.Error:
+                password = info.password
         return ConnectionInfo(
             host=info.connectionHost,
             port=info.connectionPort,
             db=info.dbName,
             user=info.username,
-            password=base64.decodebytes(info.password.encode()).decode(),
+            password=password,
             dbtype=info.serviceName,
         )
 

{deepfos-1.1.63 → deepfos-1.1.64}/deepfos/element/deepmodel.py

@@ -1,4 +1,5 @@
 import re
+import threading
 
 import numpy as np
 from asyncpg.connection import connect as pg_conn
@@ -806,36 +807,48 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         self._globals = {}
         self.before_chunk = before_chunk
         self.after_chunk = after_chunk
+        self._clients = threading.local()
 
     @future_property
-    async def client(self):
-        if self.direct_access:
-            api = await self.wait_for('async_api')
-            ver = await api.extra.version()
-            if to_version_tuple(ver, 4) >= (3, 0, 18, 8, 0):
-                db_info = await api.sharding.database()
-                space = OPTION.api.header['space']
-                if db_info.space != space:
-                    raise ValueError(
-                        f'Space id in sharding database info invalid. '
-                        f'Expected space id: {space}, actual: {db_info.space}'
-                    )
-                dbname = db_info.edgedbName
-            else:
-                dbname = None
-            client = create_async_client(
-                default_module=self.appmodule,
-                dbname=dbname
+    async def _internal_dbname(self):
+        if not self.direct_access:
+            # N.B: only retrieved when direct access is enabled
+            return
+        api = await self.wait_for('async_api')
+        ver = await api.extra.version()
+        if to_version_tuple(ver, 4) < (3, 0, 18, 8, 0):
+            return
+        db_info = await api.sharding.database()
+        space = OPTION.api.header['space']
+        if db_info.space != space:
+            raise ValueError(
+                f'Space id in sharding database info invalid. '
+                f'Expected space id: {space}, actual: {db_info.space}'
             )
-            if user_id := OPTION.api.header.get('user'):
-                default_globals = {
-                    f'{self.spacemodule}::current_user_id':
-                    user_id
-                }
-                client = client.with_globals(**default_globals)
-                self._globals = client._options.state._globals
+        return db_info.edgedbName
+
+    @property
+    def client(self):
+        if not self.direct_access:
+            return
+
+        if (client := getattr(self._clients, 'value', None)) is not None:
             return client
 
+        client = create_async_client(
+            default_module=self.appmodule,
+            dbname=self._internal_dbname
+        )
+        if user_id := OPTION.api.header.get('user'):
+            default_globals = {
+                f'{self.spacemodule}::current_user_id':
+                user_id
+            }
+            client = client.with_globals(**default_globals)
+            self._globals = client._options.state._globals
+        self._clients.value = client
+        return client
+
     @future_property
     async def element_info(self):
         """元素信息"""

{deepfos-1.1.63 → deepfos-1.1.64}/deepfos/element/workflow.py

@@ -311,9 +311,14 @@ class AsyncWorkFlow(ElementBase[WorkFlowAPI]):
                         内容一般来自文件管理的/files/upload或/files/upload/content响应值
 
 
+        Returns:
+            True: 成功
+            False: 失败
+
         .. admonition:: 示例
 
             .. code-block:: python
+
                 from deepfos.api.platform import PlatformAPI
                 from deepfos.element.workflow import WorkFlow
 
@@ -321,6 +326,7 @@ class AsyncWorkFlow(ElementBase[WorkFlowAPI]):
                 upload_resp = PlatformAPI().file.upload(
                     file_type='DL', file_name='t.txt', file='some text'
                 )
+
                 # 以动作approve，参数{"a": 1, "b": "42"}完成任务实例，
                 # 并提供附件信息为上传的't.txt'文件，备注为"Completed by SDK"
                 test_task = WorkFlow('test_task')
@@ -332,10 +338,6 @@ class AsyncWorkFlow(ElementBase[WorkFlowAPI]):
                     attachments=[upload_resp]
                 )
 
-        Returns:
-            True: 成功
-            False: 失败
-
         """
         action_id = None
         if outcome is not None:

{deepfos-1.1.63 → deepfos-1.1.64}/deepfos/lib/sysutils.py

@@ -479,14 +479,14 @@ class TaskUtil:
         py_info: 任务实例对应的python元素信息，如未提供，仍可在 `run_job_contents` 中提供
 
     """
-    api = TaskAPI(sync=True)
-    _chunksize = 200
 
     def __init__(self, task_code, py_info: PyInfo = None):
         self.task_code = task_code
+        self.api = TaskAPI(sync=True)
         if self.meta is None:
             raise ValueError(f"No config for task_code: [{self.task_code}].")
         self.py_info = py_info
+        self._chunksize = 200
 
     @cached_property
     def meta(self):
@@ -579,18 +579,17 @@ class TaskUtil:
         if not_found_col:
             raise ValueError(f'Required columns:{sorted(not_found_col)} since they are compositeKeys.')
 
-    @classmethod
-    def _create_task_instance(cls, call_api, job_contents, payload):
-        if job_contents.shape[0] <= cls._chunksize:
+    def _create_task_instance(self, call_api, job_contents, payload):
+        if job_contents.shape[0] <= self._chunksize:
             payload.jobContent = job_contents.to_dict(orient='records')
             payload.lastBatch = True
             call_api(payload)
         else:
-            payload.jobContent = job_contents.iloc[0:cls._chunksize:].to_dict(orient='records')
+            payload.jobContent = job_contents.iloc[0:self._chunksize:].to_dict(orient='records')
             payload.batchId = call_api(payload).batchId
             payloads = []
 
-            for batch_contents in split_dataframe(job_contents.iloc[cls._chunksize::], cls._chunksize):
+            for batch_contents in split_dataframe(job_contents.iloc[self._chunksize::], self._chunksize):
                 payload.jobContent = batch_contents.to_dict(orient='records')
                 payloads.append(payload)
 

{deepfos-1.1.63 → deepfos-1.1.64}/deepfos.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.63
+Version: 1.1.64
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team