deepfos 1.1.63__tar.gz → 1.1.65__tar.gz

{deepfos-1.1.63 → deepfos-1.1.65}/CHANGELOG.md

@@ -1,3 +1,18 @@
+## [1.1.65] - 2025-07-09
+
+### 更新
+
+* 修正deepmodel元素试图set client引发的问题
+
+
+## [1.1.64] - 2025-07-08
+
+### 更新
+
+* 更新连接器元素获取连接信息的接口和密码处理逻辑
+* TaskUtil的api实例化延迟至TaskUtil实例化过程中
+
+
 ## [1.1.63] - 2025-07-01
 
 ### 更新

{deepfos-1.1.63 → deepfos-1.1.65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.63
+Version: 1.1.65
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team

{deepfos-1.1.63 → deepfos-1.1.65}/deepfos/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-07-01T03:16:02+0000",
+ "date": "2025-07-09T10:58:56+0000",
  "dirty": false,
  "error": null,
- "full-revisionid": "ee27ae20cda4c8955b87eabb5b3b1c73e528e8e3",
- "version": "1.1.63"
+ "full-revisionid": "7891ba3eb0b577f318140d5ea3b613c82bda9e68",
+ "version": "1.1.65"
 }
 '''  # END VERSION_JSON
 

{deepfos-1.1.63 → deepfos-1.1.65}/deepfos/api/deepconnector.py

@@ -3,7 +3,7 @@ from typing import Union, Awaitable
 from deepfos.element.base import T_ElementInfoWithServer
 from deepfos.lib.decorator import cached_property
 
-from .base import ChildAPI, get, RootAPI
+from .base import ChildAPI, RootAPI, post
 from .models.deepconnector import *
 
 
@@ -14,10 +14,10 @@ class DataSourceAPI(ChildAPI):
     """连接信息相关接口"""
     endpoint = '/apis/v3/ds/spaces/{space}/apps/{app}'
 
-    @get('connection-info', data_wrapped=False)
+    @post('connection-info/query', data_wrapped=False)
     def connection_info(self, element_info: T_ElementInfoWithServer) -> Union[ConnectionInfoVo, Awaitable[ConnectionInfoVo]]:
         return {
-            'param': {
+            'body': {
                 'elementName': element_info.elementName,
                 'folderId': element_info.folderId
             }

{deepfos-1.1.63 → deepfos-1.1.65}/deepfos/api/models/deepconnector.py

@@ -26,3 +26,4 @@ class ConnectionInfoVo(BaseModel):
     serviceTypeName: Optional[str] = None
     serviceVersion: Optional[str] = None
     username: str
+    encryptType: Optional[str] = None

{deepfos-1.1.63 → deepfos-1.1.65}/deepfos/element/deepconnector.py

@@ -1,7 +1,10 @@
 import base64
+import binascii
+import os
 
 from deepfos.api.deepconnector import DeepConnectorAPI
 from deepfos.api.models import BaseModel
+from deepfos.db.cipher import AES
 from deepfos.element.base import ElementBase, SyncMeta
 from deepfos.lib.asynchronous import future_property
 from deepfos.lib.decorator import cached_property
@@ -9,6 +12,20 @@ from deepfos.lib.decorator import cached_property
 __all__ = ['AsyncDeepConnector', 'DeepConnector', 'ConnectionInfo']
 
 
+def decrypt(secret, cipher_text, encoding='utf8'):
+    pwd_padded = AES(secret).decrypt(
+        base64.b16decode(cipher_text)
+    )
+    if pwd_padded[-1] in range(1, 17):
+        pad_length = pwd_padded[-1]
+        pad_char = chr(pwd_padded[-1])
+        guess_pad = pad_char * pwd_padded[-1]
+
+        if pwd_padded.decode(encoding).endswith(guess_pad):
+            return pwd_padded.decode(encoding)[:-pad_length:]
+    return pwd_padded.decode(encoding)
+
+
 class ConnectionInfo(BaseModel):
     host: str
     port: int
@@ -45,12 +62,28 @@ class AsyncDeepConnector(ElementBase[DeepConnectorAPI]):
         info = await api.datasource.connection_info(
             element_info=ele_info,
         )
+        if info.encryptType == 'AES':
+            try:
+                password = decrypt(
+                    os.environ.get('EXPORT_DEEPFOS_AES_KEY', '!ABCD-EFGH-IJKL@').encode(),
+                    info.password,
+                    encoding='utf-8'
+                )
+            except ValueError:
+                raise ValueError(
+                    '连接器连接信息解密失败，请检查公共环境变量：EXPORT_DEEPFOS_AES_KEY'
+                ) from None
+        else:
+            try:
+                password = base64.decodebytes(info.password.encode()).decode()
+            except binascii.Error:
+                password = info.password
         return ConnectionInfo(
             host=info.connectionHost,
             port=info.connectionPort,
             db=info.dbName,
             user=info.username,
-            password=base64.decodebytes(info.password.encode()).decode(),
+            password=password,
             dbtype=info.serviceName,
         )
 

{deepfos-1.1.63 → deepfos-1.1.65}/deepfos/element/deepmodel.py

@@ -1,4 +1,5 @@
 import re
+import threading
 
 import numpy as np
 from asyncpg.connection import connect as pg_conn
@@ -803,39 +804,53 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             logger.debug('当前DeepModel为非直连模式')
         self.alias = AliasGenerator()
         self.pg_dsn = pg_dsn
-        self._globals = {}
+        self._globals = None
         self.before_chunk = before_chunk
         self.after_chunk = after_chunk
+        self._clients = threading.local()
 
     @future_property
-    async def client(self):
-        if self.direct_access:
-            api = await self.wait_for('async_api')
-            ver = await api.extra.version()
-            if to_version_tuple(ver, 4) >= (3, 0, 18, 8, 0):
-                db_info = await api.sharding.database()
-                space = OPTION.api.header['space']
-                if db_info.space != space:
-                    raise ValueError(
-                        f'Space id in sharding database info invalid. '
-                        f'Expected space id: {space}, actual: {db_info.space}'
-                    )
-                dbname = db_info.edgedbName
-            else:
-                dbname = None
-            client = create_async_client(
-                default_module=self.appmodule,
-                dbname=dbname
+    async def _internal_dbname(self):
+        if not self.direct_access:
+            # N.B: only retrieved when direct access is enabled
+            return
+        api = await self.wait_for('async_api')
+        ver = await api.extra.version()
+        if to_version_tuple(ver, 4) < (3, 0, 18, 8, 0):
+            return
+        db_info = await api.sharding.database()
+        space = OPTION.api.header['space']
+        if db_info.space != space:
+            raise ValueError(
+                f'Space id in sharding database info invalid. '
+                f'Expected space id: {space}, actual: {db_info.space}'
             )
-            if user_id := OPTION.api.header.get('user'):
-                default_globals = {
-                    f'{self.spacemodule}::current_user_id':
-                    user_id
-                }
-                client = client.with_globals(**default_globals)
-                self._globals = client._options.state._globals
+        return db_info.edgedbName
+
+    @property
+    def client(self):
+        if not self.direct_access:
+            self._globals = {}
+            return
+
+        if (client := getattr(self._clients, 'value', None)) is not None:
             return client
 
+        client = create_async_client(
+            default_module=self.appmodule,
+            dbname=self._internal_dbname
+        )
+        if user_id := OPTION.api.header.get('user'):
+            default_globals = {
+                f'{self.spacemodule}::current_user_id':
+                user_id
+            }
+            client = client.with_globals(**default_globals)
+        if self._globals is None:
+            self._globals = client._options.state._globals
+        self._clients.value = client
+        return client
+
     @future_property
     async def element_info(self):
         """元素信息"""
@@ -931,7 +946,9 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             logger.opt(lazy=True).debug(f"Query: [{ql}],\n"
                                         f"kwargs: [{kwargs}],\n"
                                         f"globals: [{self._globals}].")
-            _, result = await self.client.query(ql, **kwargs)
+            client = self.client
+            client._options.state._globals = self._globals
+            _, result = await client.query(ql, **kwargs)
             return result
 
         result = await self._http_query(ql, **kwargs)
@@ -972,7 +989,9 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             logger.opt(lazy=True).debug(f"Query: [{ql}],\n"
                                         f"kwargs: [{kwargs}],\n"
                                         f"globals: [{self._globals}].")
-            frame_desc, result = await self.client.query(ql, **kwargs)
+            client = self.client
+            client._options.state._globals = self._globals
+            frame_desc, result = await client.query(ql, **kwargs)
             return serutils.serialize(
                 result, ctx=serutils.Context(frame_desc=frame_desc)
             )
@@ -1019,7 +1038,9 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         
         """
         if self.direct_access:
-            frame_desc, data = await self.client.query(ql, **kwargs)
+            client = self.client
+            client._options.state._globals = self._globals
+            frame_desc, data = await client.query(ql, **kwargs)
             # set of unnamed tuple
             if isinstance(frame_desc, list):
                 return pd.DataFrame(
@@ -1061,8 +1082,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
     query_df.__doc__ = query_df.__doc__ + DOC_ARGS_KWARGS
 
     def _ensure_client(self):
-        if self.direct_access:
-            self.client # noqa
+        self.client # noqa
 
     @txn_support
     async def execute(
@@ -1139,7 +1159,9 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             return affected
 
         result = []
-        async for tx in self.client.transaction():
+        client = self.client
+        client._options.state._globals = self._globals
+        async for tx in client.transaction():
             async with tx:
                 for ql in qls_with_args:
                     logger.opt(lazy=True).debug(
@@ -1815,14 +1837,17 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             finally:
                 raise NotImplemented('非直连模式不支持设置state信息')
         else:
-            bak_cli = self.client
-            bak_globals = self.client._options.state._globals
+            if self._globals is None:
+                bak_globals = self.client._options.state._globals
+            else:
+                bak_globals = self._globals
             try:
-                self.client = self.client.with_globals(**globals_)
-                self._globals = self.client._options.state._globals
+                client = self.client
+                client._options.state._globals = bak_globals
+                client = client.with_globals(**globals_)
+                self._globals = client._options.state._globals
                 yield
             finally:
-                self.client = bak_cli
                 self._globals = bak_globals
 
     @contextmanager
@@ -1833,14 +1858,17 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             finally:
                 raise NotImplemented('非直连模式不支持设置state信息')
         else:
-            bak_cli = self.client
-            bak_globals = self.client._options.state._globals
+            if self._globals is None:
+                bak_globals = self.client._options.state._globals
+            else:
+                bak_globals = self._globals
             try:
-                self.client = self.client.without_globals(*global_names)
-                self._globals = self.client._options.state._globals
+                client = self.client
+                client._options.state._globals = bak_globals
+                client = client.without_globals(*global_names)
+                self._globals = client._options.state._globals
                 yield
             finally:
-                self.client = bak_cli
                 self._globals = bak_globals
 
 

{deepfos-1.1.63 → deepfos-1.1.65}/deepfos/element/workflow.py

@@ -311,9 +311,14 @@ class AsyncWorkFlow(ElementBase[WorkFlowAPI]):
                         内容一般来自文件管理的/files/upload或/files/upload/content响应值
 
 
+        Returns:
+            True: 成功
+            False: 失败
+
         .. admonition:: 示例
 
             .. code-block:: python
+
                 from deepfos.api.platform import PlatformAPI
                 from deepfos.element.workflow import WorkFlow
 
@@ -321,6 +326,7 @@ class AsyncWorkFlow(ElementBase[WorkFlowAPI]):
                 upload_resp = PlatformAPI().file.upload(
                     file_type='DL', file_name='t.txt', file='some text'
                 )
+
                 # 以动作approve，参数{"a": 1, "b": "42"}完成任务实例，
                 # 并提供附件信息为上传的't.txt'文件，备注为"Completed by SDK"
                 test_task = WorkFlow('test_task')
@@ -332,10 +338,6 @@ class AsyncWorkFlow(ElementBase[WorkFlowAPI]):
                     attachments=[upload_resp]
                 )
 
-        Returns:
-            True: 成功
-            False: 失败
-
         """
         action_id = None
         if outcome is not None:

{deepfos-1.1.63 → deepfos-1.1.65}/deepfos/lib/sysutils.py

@@ -479,14 +479,14 @@ class TaskUtil:
         py_info: 任务实例对应的python元素信息，如未提供，仍可在 `run_job_contents` 中提供
 
     """
-    api = TaskAPI(sync=True)
-    _chunksize = 200
 
     def __init__(self, task_code, py_info: PyInfo = None):
         self.task_code = task_code
+        self.api = TaskAPI(sync=True)
         if self.meta is None:
             raise ValueError(f"No config for task_code: [{self.task_code}].")
         self.py_info = py_info
+        self._chunksize = 200
 
     @cached_property
     def meta(self):
@@ -579,18 +579,17 @@ class TaskUtil:
         if not_found_col:
             raise ValueError(f'Required columns:{sorted(not_found_col)} since they are compositeKeys.')
 
-    @classmethod
-    def _create_task_instance(cls, call_api, job_contents, payload):
-        if job_contents.shape[0] <= cls._chunksize:
+    def _create_task_instance(self, call_api, job_contents, payload):
+        if job_contents.shape[0] <= self._chunksize:
             payload.jobContent = job_contents.to_dict(orient='records')
             payload.lastBatch = True
             call_api(payload)
         else:
-            payload.jobContent = job_contents.iloc[0:cls._chunksize:].to_dict(orient='records')
+            payload.jobContent = job_contents.iloc[0:self._chunksize:].to_dict(orient='records')
             payload.batchId = call_api(payload).batchId
             payloads = []
 
-            for batch_contents in split_dataframe(job_contents.iloc[cls._chunksize::], cls._chunksize):
+            for batch_contents in split_dataframe(job_contents.iloc[self._chunksize::], self._chunksize):
                 payload.jobContent = batch_contents.to_dict(orient='records')
                 payloads.append(payload)
 

{deepfos-1.1.63 → deepfos-1.1.65}/deepfos.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.63
+Version: 1.1.65
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team