deepfos 1.1.62__tar.gz → 1.1.64__tar.gz

{deepfos-1.1.62 → deepfos-1.1.64}/CHANGELOG.md

@@ -1,3 +1,19 @@
+## [1.1.64] - 2025-07-08
+
+### 更新
+
+* 更新连接器元素获取连接信息的接口和密码处理逻辑
+* TaskUtil的api实例化延迟至TaskUtil实例化过程中
+
+
+## [1.1.63] - 2025-07-01
+
+### 更新
+
+* 工作流完成任务实例方法增加附件列表
+* 财务模型和DeepModel批量操作支持批次前后alert
+
+
 ## [1.1.62] - 2025-06-12
 
 ### 更新

{deepfos-1.1.62 → deepfos-1.1.64}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.62
+Version: 1.1.64
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team

{deepfos-1.1.62 → deepfos-1.1.64}/deepfos/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-06-12T05:42:40+0000",
+ "date": "2025-07-08T07:07:33+0000",
  "dirty": false,
  "error": null,
- "full-revisionid": "cbd77f74d1d6b9c821c1c69be29fb027794f846c",
- "version": "1.1.62"
+ "full-revisionid": "53d949a38fa538718df6900080ebd8b0bc1358c9",
+ "version": "1.1.64"
 }
 '''  # END VERSION_JSON
 

{deepfos-1.1.62 → deepfos-1.1.64}/deepfos/api/deepconnector.py

@@ -3,7 +3,7 @@ from typing import Union, Awaitable
 from deepfos.element.base import T_ElementInfoWithServer
 from deepfos.lib.decorator import cached_property
 
-from .base import ChildAPI, get, RootAPI
+from .base import ChildAPI, RootAPI, post
 from .models.deepconnector import *
 
 
@@ -14,10 +14,10 @@ class DataSourceAPI(ChildAPI):
     """连接信息相关接口"""
     endpoint = '/apis/v3/ds/spaces/{space}/apps/{app}'
 
-    @get('connection-info', data_wrapped=False)
+    @post('connection-info/query', data_wrapped=False)
     def connection_info(self, element_info: T_ElementInfoWithServer) -> Union[ConnectionInfoVo, Awaitable[ConnectionInfoVo]]:
         return {
-            'param': {
+            'body': {
                 'elementName': element_info.elementName,
                 'folderId': element_info.folderId
             }

{deepfos-1.1.62 → deepfos-1.1.64}/deepfos/api/models/deepconnector.py

@@ -26,3 +26,4 @@ class ConnectionInfoVo(BaseModel):
     serviceTypeName: Optional[str] = None
     serviceVersion: Optional[str] = None
     username: str
+    encryptType: Optional[str] = None

{deepfos-1.1.62 → deepfos-1.1.64}/deepfos/api/models/workflow.py

@@ -10,6 +10,7 @@ generated by model_code_gen.py
 from deepfos.api.models.base import BaseModel
 from typing import List, Optional, Union, Any, Dict
 from pydantic import Field
+from deepfos.api.models.platform import FileUplodRes
 
 
 __all__ = [
@@ -576,6 +577,8 @@ class TaskCompleteInstance(BaseModel):
     extraResParams: Optional[List[TaskCompleteParam]] = None
     #: 按钮id
     actionId: Optional[str] = None
+    #: 附件文件
+    attachments: Optional[List[FileUplodRes]]
 
 
 class TaskReqDto(BaseModel):

{deepfos-1.1.62 → deepfos-1.1.64}/deepfos/element/deepconnector.py

@@ -1,7 +1,10 @@
 import base64
+import binascii
+import os
 
 from deepfos.api.deepconnector import DeepConnectorAPI
 from deepfos.api.models import BaseModel
+from deepfos.db.cipher import AES
 from deepfos.element.base import ElementBase, SyncMeta
 from deepfos.lib.asynchronous import future_property
 from deepfos.lib.decorator import cached_property
@@ -9,6 +12,20 @@ from deepfos.lib.decorator import cached_property
 __all__ = ['AsyncDeepConnector', 'DeepConnector', 'ConnectionInfo']
 
 
+def decrypt(secret, cipher_text, encoding='utf8'):
+    pwd_padded = AES(secret).decrypt(
+        base64.b16decode(cipher_text)
+    )
+    if pwd_padded[-1] in range(1, 17):
+        pad_length = pwd_padded[-1]
+        pad_char = chr(pwd_padded[-1])
+        guess_pad = pad_char * pwd_padded[-1]
+
+        if pwd_padded.decode(encoding).endswith(guess_pad):
+            return pwd_padded.decode(encoding)[:-pad_length:]
+    return pwd_padded.decode(encoding)
+
+
 class ConnectionInfo(BaseModel):
     host: str
     port: int
@@ -45,12 +62,28 @@ class AsyncDeepConnector(ElementBase[DeepConnectorAPI]):
         info = await api.datasource.connection_info(
             element_info=ele_info,
         )
+        if info.encryptType == 'AES':
+            try:
+                password = decrypt(
+                    os.environ.get('EXPORT_DEEPFOS_AES_KEY', '!ABCD-EFGH-IJKL@').encode(),
+                    info.password,
+                    encoding='utf-8'
+                )
+            except ValueError:
+                raise ValueError(
+                    '连接器连接信息解密失败，请检查公共环境变量：EXPORT_DEEPFOS_AES_KEY'
+                ) from None
+        else:
+            try:
+                password = base64.decodebytes(info.password.encode()).decode()
+            except binascii.Error:
+                password = info.password
         return ConnectionInfo(
             host=info.connectionHost,
             port=info.connectionPort,
             db=info.dbName,
             user=info.username,
-            password=base64.decodebytes(info.password.encode()).decode(),
+            password=password,
             dbtype=info.serviceName,
         )
 

{deepfos-1.1.62 → deepfos-1.1.64}/deepfos/element/deepmodel.py

@@ -1,4 +1,5 @@
 import re
+import threading
 
 import numpy as np
 from asyncpg.connection import connect as pg_conn
@@ -35,7 +36,9 @@ from deepfos.exceptions import (
 from deepfos.lib import serutils
 from deepfos.lib.asynchronous import future_property, evloop
 from deepfos.lib.decorator import flagmethod, cached_property, lru_cache
-from deepfos.lib.utils import AliasGenerator, to_version_tuple
+from deepfos.lib.utils import (
+    AliasGenerator, to_version_tuple, ChunkAlert, split_dataframe_alert
+)
 
 __all__ = ['AsyncDeepModel', 'DeepModel', 'to_fields', 'QueryWithArgs']
 
@@ -786,7 +789,13 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
     """
     __mangle_docs__ = False
 
-    def __init__(self, direct_access: bool = True, pg_dsn: str = None):  # noqa
+    def __init__(
+        self,
+        direct_access: bool = True,
+        pg_dsn: str = None,
+        before_chunk: ChunkAlert = None,
+        after_chunk: ChunkAlert = None,
+    ):
         self._txn_ = ContextVar('QLTXN')
         self.appmodule = f"app{OPTION.api.header['app']}"
         self.spacemodule = f"space{OPTION.api.header['space']}"
@@ -796,36 +805,50 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         self.alias = AliasGenerator()
         self.pg_dsn = pg_dsn
         self._globals = {}
+        self.before_chunk = before_chunk
+        self.after_chunk = after_chunk
+        self._clients = threading.local()
 
     @future_property
-    async def client(self):
-        if self.direct_access:
-            api = await self.wait_for('async_api')
-            ver = await api.extra.version()
-            if to_version_tuple(ver, 4) >= (3, 0, 18, 8, 0):
-                db_info = await api.sharding.database()
-                space = OPTION.api.header['space']
-                if db_info.space != space:
-                    raise ValueError(
-                        f'Space id in sharding database info invalid. '
-                        f'Expected space id: {space}, actual: {db_info.space}'
-                    )
-                dbname = db_info.edgedbName
-            else:
-                dbname = None
-            client = create_async_client(
-                default_module=self.appmodule,
-                dbname=dbname
+    async def _internal_dbname(self):
+        if not self.direct_access:
+            # N.B: only retrieved when direct access is enabled
+            return
+        api = await self.wait_for('async_api')
+        ver = await api.extra.version()
+        if to_version_tuple(ver, 4) < (3, 0, 18, 8, 0):
+            return
+        db_info = await api.sharding.database()
+        space = OPTION.api.header['space']
+        if db_info.space != space:
+            raise ValueError(
+                f'Space id in sharding database info invalid. '
+                f'Expected space id: {space}, actual: {db_info.space}'
             )
-            if user_id := OPTION.api.header.get('user'):
-                default_globals = {
-                    f'{self.spacemodule}::current_user_id':
-                    user_id
-                }
-                client = client.with_globals(**default_globals)
-                self._globals = client._options.state._globals
+        return db_info.edgedbName
+
+    @property
+    def client(self):
+        if not self.direct_access:
+            return
+
+        if (client := getattr(self._clients, 'value', None)) is not None:
             return client
 
+        client = create_async_client(
+            default_module=self.appmodule,
+            dbname=self._internal_dbname
+        )
+        if user_id := OPTION.api.header.get('user'):
+            default_globals = {
+                f'{self.spacemodule}::current_user_id':
+                user_id
+            }
+            client = client.with_globals(**default_globals)
+            self._globals = client._options.state._globals
+        self._clients.value = client
+        return client
+
     @future_property
     async def element_info(self):
         """元素信息"""
@@ -1288,19 +1311,22 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         qls: List[QueryWithArgs]
     ):
         self._ensure_client()
-        for i in range(0, len(data), chunksize):
-            part = data.iloc[i: i + chunksize]
-            kw_name = self.alias.get(BATCH_INSERT_KW)
-            qls.append(QueryWithArgs(
-                commands=ql.replace(
-                    f'${BATCH_INSERT_KW}', f'${kw_name}'
-                ),
-                kwargs={kw_name: part.to_json(
-                    orient='records', double_precision=15,
-                    force_ascii=False, default_handler=str
-                )},
-                globals=self._globals
-            ))
+
+        for part, alert in split_dataframe_alert(
+            data, chunksize, self.before_chunk, self.after_chunk
+        ):
+            with alert:
+                kw_name = self.alias.get(BATCH_INSERT_KW)
+                qls.append(QueryWithArgs(
+                    commands=ql.replace(
+                        f'${BATCH_INSERT_KW}', f'${kw_name}'
+                    ),
+                    kwargs={kw_name: part.to_json(
+                        orient='records', double_precision=15,
+                        force_ascii=False, default_handler=str
+                    )},
+                    globals=self._globals
+                ))
 
     @staticmethod
     def _split_self_link(data, relation, structure, bkey):

{deepfos-1.1.62 → deepfos-1.1.64}/deepfos/element/finmodel.py

@@ -25,7 +25,8 @@ from deepfos.api.app import AppAPI
 from deepfos.lib.asynchronous import future_property
 from deepfos.lib.utils import (
     unpack_expr, dict_to_expr, LazyDict, expr_to_dict,
-    dict_to_sql, split_dataframe, find_str, concat_url, CIEnumMeta
+    dict_to_sql, find_str, concat_url, CIEnumMeta, ChunkAlert,
+    split_dataframe_alert,
 )
 from deepfos.lib.sysutils import complete_cartesian_product
 from deepfos.lib.constant import (
@@ -180,11 +181,15 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         entry_object='python',
         entry_mode=1,
         server_name: Optional[str] = None,
+        before_chunk: ChunkAlert = None,
+        after_chunk: ChunkAlert = None,
     ):
         full_name = OPTION.general.task_info.get('script_name', 'python')
         self.entry_object = entry_object.format(script_name=full_name.split('.')[-1],
                                                 full_name=full_name)
         self.entry_mode = entry_mode
+        self.before_chunk = before_chunk
+        self.after_chunk = after_chunk
         super().__init__(element_name, folder_id, path, server_name)
 
     @future_property(on_demand=True)
@@ -813,30 +818,34 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
 
         # save data
         resp = []
-        for batch_data in split_dataframe(data, chunksize):
-            row_data = [
-                {"columnDimensionMemberMap": row}
-                for row in bp.dataframe_to_dict(batch_data)
-            ]
-            payload = ReactSpreadsheetSaveForm(
-                entryObject=self.entry_object,
-                sheetDatas=[SpreadsheetSingleData(
-                    cubeName=self.element_info.elementName,
-                    cubeFolderId=self.element_info.folderId,
-                    rowDatas=row_data,
-                    commonMember=pov,
-                )],
-                needCheck=need_check,
-                dataAuditSwitch=data_audit,
-                entryMode=self.entry_mode,
-                validateDimensionMember=need_check,
-                callback=callback,
-                saveDataAuthMode=auth_mode
-            )
-            r = await self.async_api.reactspreadsheet.save(
-                payload.dict(exclude_unset=True)
-            )
-            resp.append(r)
+
+        for batch_data, alert in split_dataframe_alert(
+            data, chunksize, self.before_chunk, self.after_chunk
+        ):
+            with alert:
+                row_data = [
+                    {"columnDimensionMemberMap": row}
+                    for row in bp.dataframe_to_dict(batch_data)
+                ]
+                payload = ReactSpreadsheetSaveForm(
+                    entryObject=self.entry_object,
+                    sheetDatas=[SpreadsheetSingleData(
+                        cubeName=self.element_info.elementName,
+                        cubeFolderId=self.element_info.folderId,
+                        rowDatas=row_data,
+                        commonMember=pov,
+                    )],
+                    needCheck=need_check,
+                    dataAuditSwitch=data_audit,
+                    entryMode=self.entry_mode,
+                    validateDimensionMember=need_check,
+                    callback=callback,
+                    saveDataAuthMode=auth_mode
+                )
+                r = await self.async_api.reactspreadsheet.save(
+                    payload.dict(exclude_unset=True)
+                )
+                resp.append(r)
         return resp
 
     async def delete_with_mdx(

{deepfos-1.1.62 → deepfos-1.1.64}/deepfos/element/workflow.py

@@ -4,6 +4,7 @@ from typing import *
 import pandas as pd
 
 from deepfos.api.models import compat_parse_obj_as as parse_obj_as
+from deepfos.api.models.platform import FileUplodRes
 from deepfos.api.models.workflow import *
 from deepfos.api.workflow import WorkFlowAPI
 from deepfos.element.base import ElementBase, SyncMeta
@@ -296,20 +297,47 @@ class AsyncWorkFlow(ElementBase[WorkFlowAPI]):
         file_path: str = None,
         outcome: str = None,
         extra_res_params: Dict[str, Any] = None,
+        attachments: List[Union[Dict[str, str], FileUplodRes]] = None,
     ) -> bool:
         """完成任务实例
 
         Args:
             task_id: 任务实例id
             comment: 备注
-            file_path: 附件路径
+            file_path: 附件路径(工作流版本在V3.0.4.0后弃用，后续版本请使用attachments参数)
             outcome: 结果选项，在任务可选结果不唯一时，必须提供
             extra_res_params: 完成参数
+            attachments: 附件信息列表，
+                        内容一般来自文件管理的/files/upload或/files/upload/content响应值
+
 
         Returns:
             True: 成功
             False: 失败
 
+        .. admonition:: 示例
+
+            .. code-block:: python
+
+                from deepfos.api.platform import PlatformAPI
+                from deepfos.element.workflow import WorkFlow
+
+                # 上传附件't.txt'
+                upload_resp = PlatformAPI().file.upload(
+                    file_type='DL', file_name='t.txt', file='some text'
+                )
+
+                # 以动作approve，参数{"a": 1, "b": "42"}完成任务实例，
+                # 并提供附件信息为上传的't.txt'文件，备注为"Completed by SDK"
+                test_task = WorkFlow('test_task')
+                test_task.complete_task_by_id(
+                    task_id='fd94f6a7-3467-47f9-8a3c-ff626e68dcf5',
+                    outcome='approve',
+                    extra_res_params={'a': 1, 'b': '42'},
+                    comment='Completed by SDK',
+                    attachments=[upload_resp]
+                )
+
         """
         action_id = None
         if outcome is not None:
@@ -318,8 +346,16 @@ class AsyncWorkFlow(ElementBase[WorkFlowAPI]):
                 if o.code == outcome:
                     action_id = o.id
                     break
+
         if extra_res_params is None:
             extra_res_params = {}
+        if not isinstance(extra_res_params, dict):
+            raise TypeError('extra_res_params参数应为字典类型')
+
+        if attachments is not None:
+            attachments = parse_obj_as(List[FileUplodRes], attachments)
+        else:
+            attachments = []
         return await self.async_api.task.express_complete(
             TaskCompleteInstance(
                 comment=comment, filePath=file_path, outcome=outcome, taskId=task_id,
@@ -327,7 +363,8 @@ class AsyncWorkFlow(ElementBase[WorkFlowAPI]):
                     TaskCompleteParam(name=k, value=v)
                     for k, v in extra_res_params.items()
                 ],
-                actionId=action_id
+                actionId=action_id,
+                attachments=attachments
             )
         )
 
@@ -554,7 +591,8 @@ class WorkFlow(AsyncWorkFlow, metaclass=SyncMeta):
             comment: str = None,
             file_path: str = None,
             outcome: str = None,
-            extra_res_params: Dict[str, Any] = None
+            extra_res_params: Dict[str, Any] = None,
+            attachments: List[Union[Dict[str, str], FileUplodRes]] = None,
         ) -> bool:
             ...
 

{deepfos-1.1.62 → deepfos-1.1.64}/deepfos/lib/sysutils.py

@@ -479,14 +479,14 @@ class TaskUtil:
         py_info: 任务实例对应的python元素信息，如未提供，仍可在 `run_job_contents` 中提供
 
     """
-    api = TaskAPI(sync=True)
-    _chunksize = 200
 
     def __init__(self, task_code, py_info: PyInfo = None):
         self.task_code = task_code
+        self.api = TaskAPI(sync=True)
         if self.meta is None:
             raise ValueError(f"No config for task_code: [{self.task_code}].")
         self.py_info = py_info
+        self._chunksize = 200
 
     @cached_property
     def meta(self):
@@ -579,18 +579,17 @@ class TaskUtil:
         if not_found_col:
             raise ValueError(f'Required columns:{sorted(not_found_col)} since they are compositeKeys.')
 
-    @classmethod
-    def _create_task_instance(cls, call_api, job_contents, payload):
-        if job_contents.shape[0] <= cls._chunksize:
+    def _create_task_instance(self, call_api, job_contents, payload):
+        if job_contents.shape[0] <= self._chunksize:
             payload.jobContent = job_contents.to_dict(orient='records')
             payload.lastBatch = True
             call_api(payload)
         else:
-            payload.jobContent = job_contents.iloc[0:cls._chunksize:].to_dict(orient='records')
+            payload.jobContent = job_contents.iloc[0:self._chunksize:].to_dict(orient='records')
             payload.batchId = call_api(payload).batchId
             payloads = []
 
-            for batch_contents in split_dataframe(job_contents.iloc[cls._chunksize::], cls._chunksize):
+            for batch_contents in split_dataframe(job_contents.iloc[self._chunksize::], self._chunksize):
                 payload.jobContent = batch_contents.to_dict(orient='records')
                 payloads.append(payload)
 

{deepfos-1.1.62 → deepfos-1.1.64}/deepfos/lib/utils.py

@@ -7,13 +7,14 @@ import sys
 import time
 import weakref
 from collections import UserList, UserDict, defaultdict
+from contextlib import contextmanager, nullcontext
 from enum import EnumMeta, Enum
 import random
 from typing import (
     Tuple, Optional, Dict,
     List, Union, Callable, Any,
     TypeVar, MutableMapping, Container,
-    Iterator, Iterable, DefaultDict
+    Iterator, Iterable, DefaultDict,
 )
 from itertools import groupby, count
 
@@ -867,6 +868,38 @@ def dict_to_sql(
     return sql
 
 
+class ChunkAlert:
+    def __call__(self, start: int, end: int, exc: Exception = None) -> None: ...
+
+
+@contextmanager
+def chunk_alert(
+    start: int, end: int,
+    before: ChunkAlert = None,
+    after: ChunkAlert = None,
+):
+    try:
+        if before is not None:
+            try:
+                before(start, end)
+            except Exception:
+                logger.warning('Error occurs while calling before_chunk.')
+        yield
+    except Exception as e:
+        if after is not None:
+            try:
+                after(start, end, e)
+            except Exception:
+                logger.warning('Error occurs while calling after_chunk.')
+        raise
+    else:
+        if after is not None:
+            try:
+                after(start, end)
+            except Exception:
+                logger.warning('Error occurs while calling after_chunk.')
+
+
 def split_dataframe(data: pd.DataFrame, chunksize: int = None):
     nrows = len(data)
     if chunksize is None or chunksize > nrows:
@@ -878,6 +911,33 @@ def split_dataframe(data: pd.DataFrame, chunksize: int = None):
             yield data.iloc[i: i + chunksize]
 
 
+def split_dataframe_alert(
+    data: pd.DataFrame,
+    chunksize: int = None,
+    before_chunk: ChunkAlert = None,
+    after_chunk: ChunkAlert = None,
+):
+    no_alert = before_chunk is None and after_chunk is None
+
+    nrows = len(data)
+    if chunksize is None or chunksize > nrows:
+        if no_alert:
+            yield data, nullcontext()
+        else:
+            yield data, chunk_alert(0, nrows, before_chunk, after_chunk)
+    elif chunksize <= 0:
+        raise ValueError("chunksize must be greater than 0.")
+    else:
+        for i in range(0, nrows, chunksize):
+            if no_alert:
+                yield data.iloc[i: i + chunksize], nullcontext()
+            else:
+                yield (
+                    data.iloc[i: i + chunksize],
+                    chunk_alert(i, min(i + chunksize, nrows), before_chunk, after_chunk)
+                )
+
+
 def find_str(
     target: str,
     candidates: Iterable[str],

{deepfos-1.1.62 → deepfos-1.1.64}/deepfos.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.62
+Version: 1.1.64
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team