deepfos 1.1.60__py3-none-any.whl → 1.1.78__py3-none-any.whl

deepfos/element/journal.py

@@ -1,7 +1,6 @@
 import uuid
 
-from pydantic import parse_obj_as
-
+from deepfos.api.models import compat_parse_obj_as as parse_obj_as
 from deepfos.element.datatable import get_table_class
 from deepfos.exceptions import (
     JournalModelSaveError, JournalModelCheckError, JournalModelPostingError
@@ -123,8 +122,7 @@ class AsyncJournalModel(ElementBase[JournalModelAPI]):
         line_df: pd.DataFrame,
         callback: Union[Dict, CallbackInfo] = None,
         relation_field: str = 'journal_id',
-        id_col: str = None,
-        operate_type: Literal['EDIT', 'ADD'] = 'ADD',
+        id_col: str = MAIN_ID,
         enable_create: bool = True,
         enable_default_value: bool = True,
         enable_repeat_check: bool = True,
@@ -132,9 +130,10 @@ class AsyncJournalModel(ElementBase[JournalModelAPI]):
         enable_valid_range: bool = True,
         enable_all_errors: bool = True,
         enable_need_one_line: bool = True,
+        header_operate: Literal['EDIT', 'ADD', 'DELETE_ADD'] = 'ADD',
+        line_operate: Literal['EDIT', 'ADD', 'DELETE', 'DELETE_ADD'] = 'ADD',
     ):
         errors = set()
-        is_editing = operate_type == 'EDIT'
 
         if head_df.empty:
             errors.add('凭证头表数据DataFrame不能为空')
@@ -144,6 +143,7 @@ class AsyncJournalModel(ElementBase[JournalModelAPI]):
             errors.add('凭证头行表的关联字段relation_field不能为空')
 
         self._maybe_raise_errors(errors)
+        is_editing = not (header_operate == 'ADD' and line_operate == 'ADD')
 
         if is_editing:
             head_required_fields = [relation_field, '_type', id_col]
@@ -202,7 +202,7 @@ class AsyncJournalModel(ElementBase[JournalModelAPI]):
                     index=head_data.index
                 )
             )
-        head_data = head_data.assign(operateType=operate_type)
+        head_data = head_data.assign(operateType=header_operate)
 
         # NB: replace twice in case of infer None to nan happened
         head_df = head_df.replace({None: np.nan})
@@ -228,7 +228,7 @@ class AsyncJournalModel(ElementBase[JournalModelAPI]):
         self._maybe_raise_errors(errors)
 
         line_data = line_df[line_required_fields]
-        line_data = line_data.assign(operateType=operate_type)
+        line_data = line_data.assign(operateType=line_operate)
 
         # generate lineMainId
         if is_editing:
@@ -302,7 +302,7 @@ class AsyncJournalModel(ElementBase[JournalModelAPI]):
         enable_need_one_line: bool = True,
         sync: bool = True
     ) -> CommonResultDTO:
-        """凭证模型数据保存
+        """凭证模型数据新增
         
         Args:
             head_df: 凭证头表的数据（字段名与凭证模型上头表的字段名对应）
@@ -472,6 +472,8 @@ class AsyncJournalModel(ElementBase[JournalModelAPI]):
         enable_valid_range: bool = True,
         enable_all_errors: bool = True,
         enable_need_one_line: bool = True,
+        header_operate: Literal['EDIT', 'DELETE_ADD'] = 'EDIT',
+        line_operate: Literal['EDIT', 'ADD', 'DELETE', 'DELETE_ADD'] = 'EDIT',
     ) -> CommonResultDTO:
         """凭证模型数据更新
 
@@ -492,6 +494,10 @@ class AsyncJournalModel(ElementBase[JournalModelAPI]):
             enable_valid_range: 是否启用有效性范围的校验，默认为True
             enable_all_errors: 是否启用一次性校验所有规则和数据，默认为True
             enable_need_one_line: 是否启用凭证行表至少需要一条数据的校验，默认为True
+            header_operate: 凭证头表的操作类型，默认为EDIT
+            line_operate: 凭证行表的操作类型，默认为EDIT
+                         头表EDIT, 行表可以ADD/EDIT/DELETE
+                         头表DELETE_ADD, 头表update，行表全删全增，行表只能为DELETE_ADD
 
         Returns:
            接口返回信息（CommonResultDTO的success为true 表示成功，如false 则错误在errors集合里）
@@ -501,14 +507,16 @@ class AsyncJournalModel(ElementBase[JournalModelAPI]):
         batch = self._gen_batch_payload(
             head_df=head_df, line_df=line_df,
             callback=callback, relation_field=relation_field,
-            id_col=MAIN_ID, operate_type='EDIT',
+            id_col=MAIN_ID,
             enable_create=enable_create,
             enable_default_value=enable_default_value,
             enable_repeat_check=enable_repeat_check,
             enable_required=enable_required,
             enable_valid_range=enable_valid_range,
             enable_all_errors=enable_all_errors,
-            enable_need_one_line=enable_need_one_line
+            enable_need_one_line=enable_need_one_line,
+            header_operate=header_operate,
+            line_operate=line_operate,
         )
         resp = await self.async_api.journal_model_data.update(batch)
 
@@ -815,6 +823,8 @@ class JournalModel(AsyncJournalModel, metaclass=SyncMeta):
             enable_valid_range: bool = True,
             enable_all_errors: bool = True,
             enable_need_one_line: bool = True,
+            header_operate: Literal['EDIT', 'DELETE_ADD'] = 'EDIT',
+            line_operate: Literal['EDIT', 'ADD', 'DELETE', 'DELETE_ADD'] = 'EDIT',
         ) -> CommonResultDTO:
             ...
 

deepfos/element/rolestrategy.py

@@ -24,13 +24,13 @@ __all__ = [
 # utils
 class RoleStrategyRecord(BaseModel):
     #: 用户
-    users: Optional[List[Union[UserDTO, str]]]
+    users: Optional[List[Union[UserDTO, str]]] = None
     #: 用户组
-    user_groups: Optional[List[Union[UserGroupListDTO, str]]]
+    user_groups: Optional[List[Union[UserGroupListDTO, str]]] = None
     #: 角色
-    roles: Optional[List[Union[RoleDetailVO, str]]]
+    roles: Optional[List[Union[RoleDetailVO, str]]] = None
     #: 角色组
-    role_groups: Optional[List[Union[RoleGroupDetailVO, str]]]
+    role_groups: Optional[List[Union[RoleGroupDetailVO, str]]] = None
     #: 维度表达式
     dim_expr: List[Optional[str]]
 

deepfos/element/variable.py

@@ -1,8 +1,10 @@
 from enum import Enum
-from typing import List, Dict, Union, Any, Callable, TYPE_CHECKING
+from typing import List, Dict, Union, Any, Callable, TYPE_CHECKING, Optional
 import pandas as pd
 import datetime
 
+from pydantic import Field
+
 from deepfos.api.models.variable import VariableValueDTO, VariableVO, UpdateVariavlesDTO, ValueTypeMapDTO, \
     BaseElementDetail
 from deepfos.api.variable import VariableAPI
@@ -23,13 +25,13 @@ __all__ = [
 
 
 class GlobalVariable(VariableValueDTO):
-    description: dict = {'zh-cn': None, 'en': None}
+    description: Optional[Dict] = {'zh-cn': None, 'en': None}
     scope: int = 1
     status: bool = True
 
 
 class UserVariable(VariableValueDTO):
-    description: dict = {'zh-cn': None, 'en': None}
+    description: Optional[Dict] = {'zh-cn': None, 'en': None}
     scope: int = 2
     status: bool = True
 
@@ -72,17 +74,17 @@ class NumberValueMap(ValueTypeMapDTO):
     #: 变量类型代码
     valueType: int = VariableTypeId.NUMBER
     #: 整数长度
-    maxLen: int = None
+    maxLen: Optional[int] = None
     #: 小数长度
-    digitLen: int = None
+    digitLen: Optional[int] = None
     #: 允许等于最小值
     minEqual: bool = True
     #: 最小值
-    minimun: int = None
+    minimun: Optional[int] = None
     #: 允许等于最大值
     maxEqual: bool = True
     #: 最大值
-    maximun: int = None
+    maximun: Optional[int] = None
     #: 是否显示为百分比
     percentage: bool = False
 
@@ -91,30 +93,30 @@ class SmartListValueMap(ValueTypeMapDTO):
     #: 变量类型代码
     valueType: int = VariableTypeId.SML
     #: 元素详情(由smartlist提取后的元素信息)
-    elementDetail: BaseElementDetail = None
+    elementDetail: Optional[BaseElementDetail] = None
     #: 值列表元素名称
-    valueKey: str = None
+    valueKey: Optional[str] = None
     #: 是否选择多个值列表成员
     selectedMulti: bool = False
     #: 多选值列表的成员值列表
-    valueField: List[str] = []
+    valueField: List[str] = Field(default_factory=list)
     #: 多选成员数上限
-    multipleChoiceLimit: int = None
+    multipleChoiceLimit: Optional[int] = None
 
 
 class DimensionValueMap(ValueTypeMapDTO):
     #: 变量类型代码
     valueType: int = VariableTypeId.DIM
     #: 元素详情(由Dimension提取后的元素信息)
-    elementDetail: BaseElementDetail = None
+    elementDetail: Optional[BaseElementDetail] = None
     #: 维度元素名称
-    dimensionName: str = None
+    dimensionName: Optional[str] = None
     #: 维度表达式
-    valueKey: str = None
+    valueKey: Optional[str] = None
     #: 是否选择多个值列表成员
     selectedMulti: bool = False
     #: 多选成员数上限
-    multipleChoiceLimit: int = None
+    multipleChoiceLimit: Optional[int] = None
 
 
 class DateValueMap(ValueTypeMapDTO):
@@ -128,11 +130,11 @@ class CustomListValueMap(ValueTypeMapDTO):
     #: 变量类型代码
     valueType: int = VariableTypeId.CUSTOM_LIST
     #: 自定义列表表达式(多个值请用“,”分开。示例: 1,2,3)
-    valueKey: str = None
+    valueKey: Optional[str] = None
     #: 是否选择多个值列表成员
     selectedMulti: bool = False
     #: 多选成员数上限
-    multipleChoiceLimit: int = None
+    multipleChoiceLimit: Optional[int] = None
 
 
 # -----------------------------------------------------------------------------
@@ -332,6 +334,10 @@ class AsyncVariable(ElementBase[VariableAPI]):
         if obj_hook is not None:
             return obj_hook(value)
 
+        # 启用维度表达式的维度变量应直接取原值
+        if var.valueType == VariableTypeId.DIM and not var.valueTypeMap.selectedMulti:
+            return value
+
         if not auto_cast:
             cast = None
         elif var.valueType == VariableTypeId.NUMBER:

deepfos/element/workflow.py

@@ -2,8 +2,9 @@ from functools import partial
 from typing import *
 
 import pandas as pd
-from pydantic import parse_obj_as
 
+from deepfos.api.models import compat_parse_obj_as as parse_obj_as
+from deepfos.api.models.platform import FileUplodRes
 from deepfos.api.models.workflow import *
 from deepfos.api.workflow import WorkFlowAPI
 from deepfos.element.base import ElementBase, SyncMeta
@@ -295,22 +296,76 @@ class AsyncWorkFlow(ElementBase[WorkFlowAPI]):
         comment: str = None,
         file_path: str = None,
         outcome: str = None,
+        extra_res_params: Dict[str, Any] = None,
+        attachments: List[Union[Dict[str, str], FileUplodRes]] = None,
     ) -> bool:
         """完成任务实例
 
         Args:
             task_id: 任务实例id
             comment: 备注
-            file_path: 附件路径
+            file_path: 附件路径(工作流版本在V3.0.4.0后弃用，后续版本请使用attachments参数)
             outcome: 结果选项，在任务可选结果不唯一时，必须提供
+            extra_res_params: 完成参数
+            attachments: 附件信息列表，
+                        内容一般来自文件管理的/files/upload或/files/upload/content响应值
+
 
         Returns:
             True: 成功
             False: 失败
 
+        .. admonition:: 示例
+
+            .. code-block:: python
+
+                from deepfos.api.platform import PlatformAPI
+                from deepfos.element.workflow import WorkFlow
+
+                # 上传附件't.txt'
+                upload_resp = PlatformAPI().file.upload(
+                    file_type='DL', file_name='t.txt', file='some text'
+                )
+
+                # 以动作approve，参数{"a": 1, "b": "42"}完成任务实例，
+                # 并提供附件信息为上传的't.txt'文件，备注为"Completed by SDK"
+                test_task = WorkFlow('test_task')
+                test_task.complete_task_by_id(
+                    task_id='fd94f6a7-3467-47f9-8a3c-ff626e68dcf5',
+                    outcome='approve',
+                    extra_res_params={'a': 1, 'b': '42'},
+                    comment='Completed by SDK',
+                    attachments=[upload_resp]
+                )
+
         """
+        action_id = None
+        if outcome is not None:
+            outcomes = await self.async_api.task.outcomes(task_id)
+            for o in outcomes:
+                if o.code == outcome:
+                    action_id = o.id
+                    break
+
+        if extra_res_params is None:
+            extra_res_params = {}
+        if not isinstance(extra_res_params, dict):
+            raise TypeError('extra_res_params参数应为字典类型')
+
+        if attachments is not None:
+            attachments = parse_obj_as(List[FileUplodRes], attachments)
+        else:
+            attachments = []
         return await self.async_api.task.express_complete(
-            TaskCompleteInstance(comment=comment, filePath=file_path, outcome=outcome, taskId=task_id)
+            TaskCompleteInstance(
+                comment=comment, filePath=file_path, outcome=outcome, taskId=task_id,
+                extraResParams=[
+                    TaskCompleteParam(name=k, value=v)
+                    for k, v in extra_res_params.items()
+                ],
+                actionId=action_id,
+                attachments=attachments
+            )
         )
 
     async def list_process(
@@ -536,6 +591,8 @@ class WorkFlow(AsyncWorkFlow, metaclass=SyncMeta):
             comment: str = None,
             file_path: str = None,
             outcome: str = None,
+            extra_res_params: Dict[str, Any] = None,
+            attachments: List[Union[Dict[str, str], FileUplodRes]] = None,
         ) -> bool:
             ...
 

deepfos/exceptions/__init__.py

@@ -200,7 +200,7 @@ class MsgCenterError(DeepfosBaseException):
         self.reasons = reasons
 
     def __str__(self):
-        return '推送失败，报错信息：' + '\n'.join([r.errorMsg for r in self.reasons])
+        return '推送失败，响应错误详情：\n' + '\n'.join([str(r.dict()) for r in self.reasons])
 
 
 # -----------------------------------------------------------------------------

deepfos/lib/deepchart.py

@@ -6,8 +6,9 @@ from inspect import isfunction
 from pandas import DataFrame
 from typing import *
 import pandas as pd
-from pydantic import Field, parse_obj_as
+from pydantic import Field
 
+from deepfos.api.models import compat_parse_obj_as as parse_obj_as
 from deepfos.lib.asynchronous import evloop
 from deepfos.api.models import BaseModel
 from deepfos.api.dimension import DimensionAPI
@@ -94,10 +95,10 @@ class LogicInfo(NamedTuple):
 
 
 class BaseField(BaseModel):
-    code: int = None
-    columnName: str = None
-    description: str = None
-    value_key: str = None
+    code: Optional[int] = None
+    columnName: Optional[str] = None
+    description: Optional[str] = None
+    value_key: Optional[str] = None
 
     def __init__(self, name: str = None, description: str = None, **data):
         super().__init__(columnName=name, description=description, **data)
@@ -125,14 +126,14 @@ class BaseField(BaseModel):
 
 
 class Text(BaseField):
-    code = 1
+    code: int = 1
 
 
 class Date(BaseField):
-    code = 11
+    code: int = 11
     # 前端时间筛选器颗粒度字段；4：最宽泛的颗粒度
-    value_key = '4'
-    precision = TimeType.day
+    value_key: str = '4'
+    precision: TimeType = TimeType.day
 
     @property
     def logic_info(self) -> LogicInfo:
@@ -140,11 +141,11 @@ class Date(BaseField):
 
 
 class Number(BaseField):
-    code = 15
+    code: int = 15
 
 
 class Person(BaseField):
-    code = 12
+    code: int = 12
 
 
 class BoundField(BaseField):
@@ -195,7 +196,7 @@ class BoundField(BaseField):
 
 
 class SmartList(BoundField):
-    code = 3
+    code: int = 3
     bound_class: Type[ElementBase] = SmlElement
 
     async def async_to_dict(self) -> Dict:
@@ -209,7 +210,7 @@ class SmartList(BoundField):
 
 
 class Dimension(BoundField):
-    code = 8
+    code: int = 8
     bound_class: Type[ElementBase] = DimElement
 
 

deepfos/lib/deepux.py

@@ -1,7 +1,7 @@
 """DeepUX组件数据源"""
 import asyncio
 import functools
-from typing import Dict, Type, List, Union, Tuple
+from typing import Dict, Type, List, Union, Tuple, Optional
 
 import numpy as np
 import pandas as pd
@@ -40,8 +40,8 @@ OBJECT_TYPE = "object[]"
 
 
 class BaseField(BaseModel):
-    name: str = None
-    type: str = None
+    name: Optional[str] = None
+    type: Optional[str] = None
 
     def __init__(self, name: str = None, **data):
         super().__init__(name=name, **data)
@@ -57,7 +57,7 @@ class ScalarField(BaseField):
 
 class String(ScalarField):
     """文本"""
-    type = 'str'
+    type: str = 'str'
 
     def fit(self, df: pd.DataFrame, field_name: str):
         df[field_name] = np.where(
@@ -69,7 +69,7 @@ class String(ScalarField):
 
 class Integer(ScalarField):
     """整数"""
-    type = 'int'
+    type: str = 'int'
 
     def fit(self, df: pd.DataFrame, field_name: str):
         df[field_name] = np.where(
@@ -81,7 +81,7 @@ class Integer(ScalarField):
 
 class Boolean(ScalarField):
     """布尔"""
-    type = 'bool'
+    type: str = 'bool'
 
     def fit(self, df: pd.DataFrame, field_name: str):
         df[field_name] = np.where(
@@ -93,12 +93,12 @@ class Boolean(ScalarField):
 
 class Json(ScalarField):
     """多语言文本（json）"""
-    type = 'json'
+    type: str = 'json'
 
 
 class Float(ScalarField):
     """小数"""
-    type = 'float'
+    type: str = 'float'
 
     def fit(self, df: pd.DataFrame, field_name: str):
         df[field_name] = np.where(
@@ -110,17 +110,17 @@ class Float(ScalarField):
 
 class DateTime(ScalarField):
     """日期时间"""
-    type = 'datetime'
+    type: str = 'datetime'
 
 
 class UUID(ScalarField):
     """uuid"""
-    type = 'uuid'
+    type: str = 'uuid'
 
 
 class ObjectField(BaseField):
     """表示对象类型的结构信息, 图格式数据源专用"""
-    type = OBJECT_TYPE
+    type: str = OBJECT_TYPE
     fields: List[ScalarField] = Field(default_factory=list)
 
 

deepfos/lib/discovery.py

@@ -256,6 +256,9 @@ class ServiceDiscovery(ABC):
             elif impl == 'nacos':
                 from deepfos.lib.nacos import Nacos
                 cls.__ins__[impl] = Nacos
+            elif impl == 'k8s':
+                from deepfos.lib.k8s import K8s
+                cls.__ins__[impl] = K8s
 
         return cls.__ins__[impl]
 

deepfos/lib/filterparser.py

@@ -2,9 +2,9 @@ import collections
 import re
 from enum import Enum, IntEnum
 from typing import *
-from pydantic import Field, parse_obj_as
+from pydantic import Field
 
-from deepfos.api.models import BaseModel
+from deepfos.api.models import BaseModel, compat_parse_obj_as as parse_obj_as
 from deepfos.core.logictable.nodemixin import NodeMixin, TreeRenderer
 from deepfos.lib.utils import CIEnum, CIEnumMeta
 

deepfos/lib/k8s.py

@@ -0,0 +1,101 @@
+import asyncio
+from collections import defaultdict
+from functools import cached_property
+from typing import TypedDict, Literal, Type, Dict, FrozenSet
+
+from loguru import logger
+from kubernetes_asyncio import client, config
+from kubernetes_asyncio.client import (
+    V1ServiceList,
+    V1Endpoints,
+    V1EndpointSubset,
+    V1EndpointAddress,
+    CoreV1EndpointPort,
+)
+from deepfos.lib.discovery import ServiceDiscovery
+from deepfos import OPTION
+
+
+class K8s(ServiceDiscovery):
+    def __init__(self):
+        super().__init__()
+        self._instance_lock: Dict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
+        self._server_lock = asyncio.Lock()
+        self._cares = set()
+        self._ns = OPTION.k8s.namespace
+
+    async def on_close(self):
+        self._cares.clear()
+        self.server_cache.clear()
+        self._instance_lock.clear()
+
+    async def on_startup(self):
+        config.load_incluster_config()
+
+    @cached_property
+    def _api(self):
+        return client.CoreV1Api()
+
+    async def on_interval(self):
+        await self._update_cache()
+
+    async def _update_cache(self):
+        logger.opt(lazy=True).debug(f"Update cache for instance: {self._cares}")
+        await asyncio.gather(*(
+            self.update_instance_for_service(sn)
+            for sn in self._cares
+        ))
+
+    async def update_service_cache(self, server_name: str):
+        async with self._server_lock:
+            if server_name not in self.server_cache:
+                await self.update_services()
+
+    async def update_instance_cache(self, server_name):
+        self._cares.add(server_name)
+        await self.update_instance_for_service(server_name)
+
+
+    async def _list_all_service_names(self) -> FrozenSet[str]:
+        svcs: V1ServiceList = await self._api.list_namespaced_service(self._ns)
+        return frozenset(svc.metadata.name for svc in svcs.items)
+
+    async def update_services(self):
+        new_services = await self._list_all_service_names()
+        cur_services = frozenset(self.server_cache.keys())
+
+        if added := new_services - cur_services:
+            for srv in added:
+                self.server_cache.__getitem__(srv)
+            logger.debug(f"Added services: {added}")
+
+        if removed := cur_services - new_services:
+            for srv in removed:
+                self.server_cache.pop(srv)
+                self._instance_lock.pop(srv, None)  # noqa
+            logger.debug(f"Removed servieces: {removed}")
+
+    async def update_instance_for_service(self, server_name: str):
+        endpoints: V1Endpoints = await self._api.read_namespaced_endpoints(
+            server_name,
+            self._ns,
+        )
+
+        no_active_nodes = True
+        async with self._instance_lock[server_name]:
+            cache = self.server_cache[server_name]
+
+            for subset in endpoints.subsets or []:
+                subset: V1EndpointSubset
+                for addr in subset.addresses or []:
+                    addr: V1EndpointAddress
+                    no_active_nodes = False
+                    for port in subset.ports:
+                        port: CoreV1EndpointPort
+                        url = f"{port.name}://{addr.ip}:{port.port}"
+                        cache.add(url)
+
+            if no_active_nodes:
+                logger.debug(f"{server_name} has no active endpoint, clear invalid...")
+                for item in list(cache):
+                    cache.delete(item)