deepfos 1.1.60__py3-none-any.whl → 1.1.78__py3-none-any.whl

deepfos/api/platform.py

@@ -93,7 +93,8 @@ class StdFileAPI(ChildAPI):
     
     
         """
-        data = FormData()
+        # platform server does not recognize 7bit encoding
+        data = FormData(quote_fields=False)
         data.add_field('file', file, filename=file_name)
         return {'body': data,
                 'param': {
@@ -174,6 +175,7 @@ class FilePyAPI(ChildAPI):
 class PlatformAPI(RootAPI):
     """文件上传服务接口"""
     prefix = lambda: OPTION.server.platform_file
+    server_name = "platform-file-server"
 
     @cached_property
     def file(self) -> StdFileAPI:

deepfos/api/space.py

@@ -522,6 +522,7 @@ class SpaceInternalAPI(ChildAPI):
 class SpaceAPI(RootAPI):
     """空间服务接口"""
     prefix = lambda: OPTION.server.space
+    server_name = "space-server"
 
     @cached_property
     def app(self) -> SpaceAppAPI:

deepfos/api/system.py

@@ -312,6 +312,7 @@ class Space(ChildAPI):
 class SystemAPI(RootAPI):
     """系统服务接口"""
     prefix = lambda: OPTION.server.system
+    server_name = "system-server"
 
     @cached_property
     def data_source(self) -> SysDataSourceAPI:

deepfos/api/workflow.py

@@ -188,6 +188,14 @@ class ProcessTask(ChildAPI):
         """
         return {'body': userTaskQueryDTO}
 
+    @get('outcomes')
+    def outcomes(self, taskId: str) -> Union[List[OutComeDto], Awaitable[List[OutComeDto]]]:
+        """
+        获取用户任务outcome
+
+        """
+        return {'param': {'taskId': taskId}}
+
 
 class Version(ChildAPI):
     endpoint = '/version'

deepfos/cache.py

@@ -106,16 +106,62 @@ class SpaceSeperatedBase(CustomedCache):
 
 
 class AppSeperatedTTLCache(TTLCache, AppSeperatedBase):
-    pass
+    def popitem(self):
+        """Remove and return the `(key, value)` pair least recently used that
+        has not already expired.
+
+        """
+        with self.timer as time:
+            self.expire(time)
+            try:
+                key = next(iter(self._TTLCache__links))
+            except StopIteration:
+                raise KeyError("%s is empty" % type(self).__name__) from None
+            else:
+                value = TTLCache.__getitem__(self, key)
+                TTLCache.__delitem__(self, key)
+                return key, value
 
 
 class SpaceSeperatedTTLCache(TTLCache, SpaceSeperatedBase):
-    pass
+    def popitem(self):
+        """Remove and return the `(key, value)` pair least recently used that
+        has not already expired.
+
+        """
+        with self.timer as time:
+            self.expire(time)
+            try:
+                key = next(iter(self._TTLCache__links))
+            except StopIteration:
+                raise KeyError("%s is empty" % type(self).__name__) from None
+            else:
+                value = TTLCache.__getitem__(self, key)
+                TTLCache.__delitem__(self, key)
+                return key, value
 
 
 class AppSeperatedLRUCache(LRUCache, AppSeperatedBase):
-    pass
+    def popitem(self):
+        """Remove and return the `(key, value)` pair least recently used."""
+        try:
+            key = next(iter(self._LRUCache__order))
+        except StopIteration:
+            raise KeyError("%s is empty" % type(self).__name__) from None
+        else:
+            value = LRUCache.__getitem__(self, key)
+            LRUCache.__delitem__(self, key)
+            return key, value
 
 
 class SpaceSeperatedLRUCache(LRUCache, SpaceSeperatedBase):
-    pass
+    def popitem(self):
+        """Remove and return the `(key, value)` pair least recently used."""
+        try:
+            key = next(iter(self._LRUCache__order))
+        except StopIteration:
+            raise KeyError("%s is empty" % type(self).__name__) from None
+        else:
+            value = LRUCache.__getitem__(self, key)
+            LRUCache.__delitem__(self, key)
+            return key, value

deepfos/element/bizmodel.py

@@ -57,8 +57,8 @@ def dataframe_to_records(
 
 
 class TableNode(MetaTable):
-    table_structure: TableStructure = None
-    name: str = None
+    table_structure: Optional[TableStructure] = None
+    name: Optional[str] = None
 
 
 def create_table(struct: TableStructure, parent: BaseTable = None) -> TableNode:

deepfos/element/deep_pipeline.py

@@ -1,17 +1,19 @@
 import asyncio
 import time
+import warnings
 from math import ceil
 from typing import Any, TYPE_CHECKING, List
 
 from deepfos import OPTION
 from deepfos.api.deep_pipeline import DeepPipelineAPI
-from deepfos.api.models.deep_pipeline import RunInfo, FlowInfo
+from deepfos.api.models.deep_pipeline import RunInfo, FlowInfo, RunInfoWithParam
 from deepfos.element.base import ElementBase, SyncMeta
 from deepfos.exceptions import (
     APIResponseError, RunIdInvalid, RunTerminated,
     ReleaseFlowTimeout, RunFailedError, ReleaseFlowNotExists,
 )
 from deepfos.lib.asynchronous import future_property
+from deepfos.lib.constant import UNSET
 from deepfos.lib.decorator import cached_property
 
 errcode_map = {
@@ -47,7 +49,7 @@ class AsyncDeepPipeline(ElementBase[DeepPipelineAPI]):
 
     async def run(
         self,
-        parameter: Any = None,
+        parameter: Any = UNSET,
         timeout: int = None,
         in_process: bool = True,
     ) -> Any:
@@ -57,25 +59,29 @@ class AsyncDeepPipeline(ElementBase[DeepPipelineAPI]):
         Args:
             parameter: 执行参数
             timeout: 超时时间(秒)
-            in_process: 是否在同一个进程执行
+            in_process: 是否在同一个进程执行（仅作新旧兼容，实际只会在同一个进程执行，即不带启停启动）
 
         Returns:
             执行结果
         """
-        run_id = await self.run_async(parameter, in_process)
+        if not in_process:
+            warnings.warn(
+                '同步启动固定为同一个进程执行(不带启停启动)，in_process参数为False不会影响该行为'
+            )
+        run_id = await self.run_async(parameter, True)
         return await self.result(run_id, timeout)
 
     async def run_async(
         self,
-        parameter: Any = None,
-        in_process: bool = True,
+        parameter: Any = UNSET,
+        in_process: bool = False,
     ) -> str:
         """
         异步启动数据流
 
         Args:
             parameter: 执行参数
-            in_process: 是否在同一个进程执行
+            in_process: 是否在同一个进程执行，默认False，即带启停启动
 
         Returns:
             执行ID
@@ -83,14 +89,21 @@ class AsyncDeepPipeline(ElementBase[DeepPipelineAPI]):
         if not self.has_approved_release:
             raise ReleaseFlowNotExists('暂无启用中状态的数据流版本')
 
-        return await self.async_api.run.run_async(
-            RunInfo(
+        if parameter is UNSET:
+            payload = RunInfo(
+                elementName=self.element_name,
+                folderId=self.element_info.folderId,
+                inProcess=in_process
+            )
+        else:
+            payload = RunInfoWithParam(
                 elementName=self.element_name,
                 parameter=parameter,
                 folderId=self.element_info.folderId,
                 inProcess=in_process
             )
-        )
+
+        return await self.async_api.run.run_async(payload)
 
     async def result(self, run_id: str, timeout: int = None) -> Any:
         """
@@ -134,14 +147,14 @@ class AsyncDeepPipeline(ElementBase[DeepPipelineAPI]):
     async def run_batch(
         self,
         parameters: List[Any],
-        in_process: bool = True,
+        in_process: bool = False,
     ) -> List[str]:
         """
         批量异步启动数据流
 
         Args:
             parameters: 执行参数列表
-            in_process: 是否在同一个进程执行
+            in_process: 是否在同一个进程执行，默认False，即带启停启动
 
         Returns:
             执行ID列表
@@ -162,7 +175,7 @@ class DeepPipeline(AsyncDeepPipeline, metaclass=SyncMeta):
     if TYPE_CHECKING:  # pragma: no cover
         def run(
             self,
-            parameter: Any = None,
+            parameter: Any = UNSET,
             timeout: int = None,
             in_process: bool = True,
         ) -> Any:
@@ -170,8 +183,8 @@ class DeepPipeline(AsyncDeepPipeline, metaclass=SyncMeta):
 
         def run_async(
             self,
-            parameter: Any = None,
-            in_process: bool = True,
+            parameter: Any = UNSET,
+            in_process: bool = False,
         ) -> str:
             ...
 
@@ -181,6 +194,6 @@ class DeepPipeline(AsyncDeepPipeline, metaclass=SyncMeta):
         def run_batch(
             self,
             parameters: List[Any],
-            in_process: bool = True,
+            in_process: bool = False,
         ):
             ...

deepfos/element/deepconnector.py

@@ -1,7 +1,10 @@
 import base64
+import binascii
+import os
 
 from deepfos.api.deepconnector import DeepConnectorAPI
 from deepfos.api.models import BaseModel
+from deepfos.db.cipher import AES
 from deepfos.element.base import ElementBase, SyncMeta
 from deepfos.lib.asynchronous import future_property
 from deepfos.lib.decorator import cached_property
@@ -9,6 +12,22 @@ from deepfos.lib.decorator import cached_property
 __all__ = ['AsyncDeepConnector', 'DeepConnector', 'ConnectionInfo']
 
 
+def decrypt(secret, cipher_text, encoding='utf8'):
+    pwd_padded = b''
+    for i in range(len(cipher_text))[::32]:
+        pwd_padded += AES(secret).decrypt(
+            base64.b16decode(cipher_text[i: i + 32])
+        )
+    if pwd_padded[-1] in range(1, 17):
+        pad_length = pwd_padded[-1]
+        pad_char = chr(pwd_padded[-1])
+        guess_pad = pad_char * pwd_padded[-1]
+
+        if pwd_padded.decode(encoding).endswith(guess_pad):
+            return pwd_padded.decode(encoding)[:-pad_length:]
+    return pwd_padded.decode(encoding)
+
+
 class ConnectionInfo(BaseModel):
     host: str
     port: int
@@ -45,12 +64,28 @@ class AsyncDeepConnector(ElementBase[DeepConnectorAPI]):
         info = await api.datasource.connection_info(
             element_info=ele_info,
         )
+        if info.encryptType == 'AES':
+            try:
+                password = decrypt(
+                    os.environ.get('EXPORT_DEEPFOS_AES_KEY', '!ABCD-EFGH-IJKL@').encode(),
+                    info.password,
+                    encoding='utf-8'
+                )
+            except ValueError:
+                raise ValueError(
+                    '连接器连接信息解密失败，请检查公共环境变量：EXPORT_DEEPFOS_AES_KEY'
+                ) from None
+        else:
+            try:
+                password = base64.decodebytes(info.password.encode()).decode()
+            except binascii.Error:
+                password = info.password
         return ConnectionInfo(
             host=info.connectionHost,
             port=info.connectionPort,
             db=info.dbName,
             user=info.username,
-            password=base64.decodebytes(info.password.encode()).decode(),
+            password=password,
             dbtype=info.serviceName,
         )