deepfos 1.1.60__py3-none-any.whl → 1.1.78__py3-none-any.whl

deepfos/lib/msg.py

@@ -110,8 +110,9 @@ class AsyncMsgCenter:
         title_param: Dict[str, str] = None,
         content_param: Dict[str, str] = None,
         attachments: Dict[str, Union[str, bytes]] = None,
-        cc_email: List[str] = None,
-        api_idx: Literal[0, 1] = 0
+        cc_email: List = None,
+        api_idx: Literal[0, 1] = 0,
+        success_rule: Literal['partial', 'all'] = 'all'
     ) -> List:
         payload = {
             'receiver': receivers,
@@ -139,7 +140,11 @@ class AsyncMsgCenter:
 
         api = self.type_api_map[tpl_type][api_idx]
         resp = await api(payload)
-        if resp.failure:
+        if resp.failure and (
+            success_rule == 'all' or (
+                success_rule == 'partial' and not resp.success
+            )
+        ):
             raise MsgCenterError(*resp.failure)
         
         return resp.success
@@ -153,6 +158,9 @@ class AsyncMsgCenter:
         title_param: Dict[str, str] = None,
         content_param: Dict[str, str] = None,
         attachments: Dict[str, Union[str, bytes]] = None,
+        success_rule: Literal['partial', 'all'] = 'all',
+        cc_users: List[str] = None,
+        cc_groups: List[str] = None,
     ) -> List:
         """推送指定消息模版的消息
 
@@ -164,6 +172,9 @@ class AsyncMsgCenter:
             title_param: 可选，标题变量
             content_param: 可选，内容变量
             attachments: 可选，站内消息或邮箱的消息附件，以 文件名: 文件(字符串/bytes) 的字典形式提供
+            success_rule: 可选，发送成功规则，'partial' 表示部分成功即可，'all' 表示所有收件人都必须成功
+            cc_users: 可选，抄送人userid列表
+            cc_groups: 可选，抄送人groupid列表
 
         .. admonition:: 示例
 
@@ -210,6 +221,11 @@ class AsyncMsgCenter:
             receivers.extend([{'id': r, 'type': 'USER'} for r in receiver_users])
         if receiver_groups:
             receivers.extend([{'id': r, 'type': 'GROUP'} for r in receiver_groups])
+        cc_email = []
+        if cc_users:
+            cc_email.extend([{'id': r, 'type': 'USER'} for r in cc_users])
+        if cc_groups:
+            cc_email.extend([{'id': r, 'type': 'GROUP'} for r in cc_groups])
 
         if not receivers:
             raise ValueError('需提供receiver_users和receiver_groups中的至少一项')
@@ -217,7 +233,8 @@ class AsyncMsgCenter:
         return await self._publish(
             template, sender=sender, receivers=receivers,
             title_param=title_param, content_param=content_param,
-            attachments=attachments,
+            attachments=attachments, success_rule=success_rule,
+            cc_email=cc_email
         )
 
     async def send_mail(
@@ -229,6 +246,7 @@ class AsyncMsgCenter:
         content_param: Dict[str, str] = None,
         attachments: Dict[str, Union[str, bytes]] = None,
         cc_email: List[str] = None,
+        success_rule: Literal['partial', 'all'] = 'all'
     ) -> List:
         """发送指定消息模版的邮件
 
@@ -240,6 +258,7 @@ class AsyncMsgCenter:
             content_param: 可选，内容变量
             attachments: 可选，附件，以 文件名: 文件(字符串/bytes) 的字典形式提供
             cc_email: 可选，抄送人列表
+            success_rule: 可选，发送成功规则，'partial' 表示部分成功即可，'all' 表示所有收件人都必须成功
 
 
         .. admonition:: 示例
@@ -289,7 +308,7 @@ class AsyncMsgCenter:
             template, sender=sender, receivers=receivers,
             title_param=title_param, content_param=content_param,
             attachments=attachments, cc_email=cc_email,
-            api_idx=1
+            api_idx=1, success_rule=success_rule,
         )
 
     async def send_sms(
@@ -298,7 +317,8 @@ class AsyncMsgCenter:
         receivers: List[str],
         sender: str = None,
         title_param: Dict[str, str] = None,
-        content_param: Dict[str, str] = None
+        content_param: Dict[str, str] = None,
+        success_rule: Literal['partial', 'all'] = 'all'
     ) -> List:
         """发送指定消息模版的短信
 
@@ -308,6 +328,7 @@ class AsyncMsgCenter:
             sender: 可选，发送人userid，默认为当前用户id
             title_param: 可选，标题变量
             content_param: 可选，内容变量
+            success_rule: 可选，发送成功规则，'partial' 表示部分成功即可，'all' 表示所有收件人都必须成功
 
 
         .. admonition:: 示例
@@ -351,7 +372,7 @@ class AsyncMsgCenter:
         return await self._publish(
             template, sender=sender, receivers=receivers,
             title_param=title_param, content_param=content_param,
-            api_idx=1
+            api_idx=1, success_rule=success_rule,
         )
 
 
@@ -368,6 +389,9 @@ class MsgCenter(AsyncMsgCenter, metaclass=SyncMeta):
             title_param: Dict[str, str] = None,
             content_param: Dict[str, str] = None,
             attachments: Dict[str, Union[str, bytes]] = None,
+            success_rule: Literal['partial', 'all'] = 'all',
+            cc_users: List[str] = None,
+            cc_groups: List[str] = None,
         ) -> List:
             ...
 
@@ -380,6 +404,7 @@ class MsgCenter(AsyncMsgCenter, metaclass=SyncMeta):
             content_param: Dict[str, str] = None,
             attachments: Dict[str, Union[str, bytes]] = None,
             cc_email: List[str] = None,
+            success_rule: Literal['partial', 'all'] = 'all'
         ) -> List:
             ...
 
@@ -389,6 +414,7 @@ class MsgCenter(AsyncMsgCenter, metaclass=SyncMeta):
             receivers: List[str],
             sender: str = None,
             title_param: Dict[str, str] = None,
-            content_param: Dict[str, str] = None
+            content_param: Dict[str, str] = None,
+            success_rule: Literal['partial', 'all'] = 'all'
         ) -> List:
             ...

deepfos/lib/serutils.py

@@ -3,9 +3,10 @@ from __future__ import annotations
 import datetime
 import decimal
 import functools
+import json
 import uuid
 from dataclasses import dataclass
-from typing import Any
+from typing import Any, Dict, Union, List
 
 import edgedb
 from edgedb.introspect import introspect_object as intro
@@ -24,12 +25,20 @@ def serialize(o, ctx: Context = Context()):
 
 
 @functools.singledispatch
-def deserialize(o):
+def deserialize(o, field_info: Union[Dict, List[str], str] = None):
     raise TypeError(f'无法反序列化类型: {type(o)}')
 
 
 @deserialize.register
-def to_object(o: dict):
+def to_object(o: dict, field_info: Union[Dict, List[str], str] = None):
+    if field_info == 'json':
+        return o
+    if (
+        isinstance(field_info, dict)
+        and len(field_info) == 1
+        and field_info.get(None) == 'json'
+    ):
+        return o
     _id = o.pop('id', None)
     ordered_attr = o.keys()
     obj_cls = datatypes.create_object_factory(
@@ -42,20 +51,34 @@ def to_object(o: dict):
             actual_id = uuid.UUID(_id)
         elif isinstance(_id, uuid.UUID):
             actual_id = _id
+    # NB: in case of incorrect field info
+    if not isinstance(field_info, dict):
+        field_info = {}
+
     return obj_cls(
         actual_id,
-        *[deserialize(o[k]) for k in ordered_attr]
+        *[deserialize(o[k], field_info.get(k, {})) for k in ordered_attr]
     )
 
 
 @deserialize.register
-def to_set(o: list):
-    return [deserialize(ele) for ele in o]
+def to_set(o: list, field_info: Union[Dict, List[str], str] = None):
+    if isinstance(field_info, list):
+        field_info = {
+            f['name']: f['type'] if f['fields'] is None else f['fields']
+            for f in field_info
+        }
+    return [deserialize(ele, field_info) for ele in o]
 
 
 @deserialize.register
-def to_tuple(o: tuple):
-    return tuple(deserialize(ele) for ele in o)
+def to_tuple(o: tuple, field_info: Union[Dict, List[str], str] = None):
+    if isinstance(field_info, list):
+        field_info = {
+            f['name']: f['type'] if f['fields'] is None else f['fields']
+            for f in field_info
+        }
+    return tuple(deserialize(ele, field_info) for ele in o)
 
 
 @deserialize.register(int)
@@ -65,7 +88,7 @@ def to_tuple(o: tuple):
 @deserialize.register(bool)
 @deserialize.register(type(None))
 @deserialize.register(datetime.datetime)
-def to_scalar(o):
+def to_scalar(o, field_info: Union[Dict, List[str], str] = None):
     return o
 
 
@@ -173,6 +196,8 @@ def _set(o, ctx: Context = Context()):
 @serialize.register(uuid.UUID)
 @serialize.register(decimal.Decimal)
 def _scalar(o, ctx: Context = Context()):
+    if ctx.frame_desc == 'std::json' and isinstance(o, str):
+        return json.loads(o)
     return o
 
 

deepfos/lib/sysutils.py

@@ -104,6 +104,8 @@ def complete_cartesian_product(
     fix: Dict[str, Union[str, list]],
     df: pd.DataFrame = None,
     paths: Union[str, Dict[str, str]] = None,
+    folder_ids: Union[str, Dict[str, str]] = None,
+    col_dim_map: Dict[str, str] = None,
 ) -> pd.DataFrame:
     """
     构造完整的维度成员笛卡尔积
@@ -113,6 +115,8 @@ def complete_cartesian_product(
         df: 如果需要为现有DataFrame补全笛卡尔积，传入一个df。如果不传，则是生成fix中维度成员的笛卡尔积。
         paths: fix中维度的path，如果所有维度的目录相同，传同一个path，否则传字典，key为维度名，value为path。
           如果不传，则自动寻找维度对应的path。
+        folder_ids: 类似paths, 但值是folder_id
+        col_dim_map: data中的列名与实际维度名的映射关系，默认data中的列名与维度名相同
 
     Returns:
         维度成员笛卡尔积的DataFrame
@@ -147,33 +151,49 @@ def complete_cartesian_product(
     from deepfos.element.dimension import AsyncDimension
     from .asynchronous import evloop
 
-    if isinstance(paths, dict):
-        path_getter = paths.__getitem__
+    if paths is not None:
+        loc_key = "path"
+        if isinstance(paths, dict):
+            loc_getter = paths.get
+        else:
+            loc_getter = lambda _: paths
+    elif folder_ids is not None:
+        loc_key = "folder_id"
+
+        if isinstance(folder_ids, dict):
+            loc_getter = folder_ids.get
+        else:
+            loc_getter = lambda _: folder_ids
     else:
-        path_getter = lambda _: paths
+        loc_key = "path"
+        loc_getter = lambda _: None
+    if col_dim_map is None:
+        col_dim_map = {}
 
     # 遍历fix，如果fix的值为str，则认为是维度表达式，将表达式转换为成员list
     mbrs = {}
     futures = []
 
-    for dim, exp in fix.items():
+    for col, exp in fix.items():
         if isinstance(exp, str):
             if "(" not in exp:
-                mbrs[dim] = exp.split(';')
+                mbrs[col] = exp.split(';')
             else:
-                path = path_getter(dim)
-                future = evloop.apply(AsyncDimension(element_name=dim, path=path).query(
+                dim = col_dim_map.get(col, col)
+                loc = loc_getter(dim)
+                future = evloop.apply(AsyncDimension(element_name=dim, **{loc_key: loc}).query(
                     expression=exp, fields=['name'], as_model=False
                 ))
 
-                futures.append((dim, future))
+                futures.append((col, future))
         else:
             if not isinstance(exp, list):
                 raise TypeError('fix参数的value只能为维度表达式(str)或维度成员(list)')
-            mbrs[dim] = exp
+            mbrs[col] = exp
 
-    for dim, future in futures:
-        mbrs[dim] = [item['name'] for item in future.result()]
+    for col, future in futures:
+        dim_mbrs = future.result()
+        mbrs[col] = [item['name'] for item in dim_mbrs]
 
     if df is None:
         df = pd.DataFrame(columns=list(fix.keys()))
@@ -459,14 +479,14 @@ class TaskUtil:
         py_info: 任务实例对应的python元素信息，如未提供，仍可在 `run_job_contents` 中提供
 
     """
-    api = TaskAPI(sync=True)
-    _chunksize = 200
 
     def __init__(self, task_code, py_info: PyInfo = None):
         self.task_code = task_code
+        self.api = TaskAPI(sync=True)
         if self.meta is None:
             raise ValueError(f"No config for task_code: [{self.task_code}].")
         self.py_info = py_info
+        self._chunksize = 200
 
     @cached_property
     def meta(self):
@@ -559,18 +579,17 @@ class TaskUtil:
         if not_found_col:
             raise ValueError(f'Required columns:{sorted(not_found_col)} since they are compositeKeys.')
 
-    @classmethod
-    def _create_task_instance(cls, call_api, job_contents, payload):
-        if job_contents.shape[0] <= cls._chunksize:
+    def _create_task_instance(self, call_api, job_contents, payload):
+        if job_contents.shape[0] <= self._chunksize:
             payload.jobContent = job_contents.to_dict(orient='records')
             payload.lastBatch = True
             call_api(payload)
         else:
-            payload.jobContent = job_contents.iloc[0:cls._chunksize:].to_dict(orient='records')
+            payload.jobContent = job_contents.iloc[0:self._chunksize:].to_dict(orient='records')
             payload.batchId = call_api(payload).batchId
             payloads = []
 
-            for batch_contents in split_dataframe(job_contents.iloc[cls._chunksize::], cls._chunksize):
+            for batch_contents in split_dataframe(job_contents.iloc[self._chunksize::], self._chunksize):
                 payload.jobContent = batch_contents.to_dict(orient='records')
                 payloads.append(payload)
 

deepfos/lib/utils.py

@@ -7,13 +7,14 @@ import sys
 import time
 import weakref
 from collections import UserList, UserDict, defaultdict
+from contextlib import contextmanager, nullcontext
 from enum import EnumMeta, Enum
 import random
 from typing import (
     Tuple, Optional, Dict,
     List, Union, Callable, Any,
     TypeVar, MutableMapping, Container,
-    Iterator, Iterable, DefaultDict
+    Iterator, Iterable, DefaultDict,
 )
 from itertools import groupby, count
 
@@ -867,6 +868,38 @@ def dict_to_sql(
     return sql
 
 
+class ChunkAlert:
+    def __call__(self, start: int, end: int, exc: Exception = None) -> None: ...
+
+
+@contextmanager
+def chunk_alert(
+    start: int, end: int,
+    before: ChunkAlert = None,
+    after: ChunkAlert = None,
+):
+    try:
+        if before is not None:
+            try:
+                before(start, end)
+            except Exception:
+                logger.warning('Error occurs while calling before_chunk.')
+        yield
+    except Exception as e:
+        if after is not None:
+            try:
+                after(start, end, e)
+            except Exception:
+                logger.warning('Error occurs while calling after_chunk.')
+        raise
+    else:
+        if after is not None:
+            try:
+                after(start, end)
+            except Exception:
+                logger.warning('Error occurs while calling after_chunk.')
+
+
 def split_dataframe(data: pd.DataFrame, chunksize: int = None):
     nrows = len(data)
     if chunksize is None or chunksize > nrows:
@@ -878,6 +911,33 @@ def split_dataframe(data: pd.DataFrame, chunksize: int = None):
             yield data.iloc[i: i + chunksize]
 
 
+def split_dataframe_alert(
+    data: pd.DataFrame,
+    chunksize: int = None,
+    before_chunk: ChunkAlert = None,
+    after_chunk: ChunkAlert = None,
+):
+    no_alert = before_chunk is None and after_chunk is None
+
+    nrows = len(data)
+    if chunksize is None or chunksize > nrows:
+        if no_alert:
+            yield data, nullcontext()
+        else:
+            yield data, chunk_alert(0, nrows, before_chunk, after_chunk)
+    elif chunksize <= 0:
+        raise ValueError("chunksize must be greater than 0.")
+    else:
+        for i in range(0, nrows, chunksize):
+            if no_alert:
+                yield data.iloc[i: i + chunksize], nullcontext()
+            else:
+                yield (
+                    data.iloc[i: i + chunksize],
+                    chunk_alert(i, min(i + chunksize, nrows), before_chunk, after_chunk)
+                )
+
+
 def find_str(
     target: str,
     candidates: Iterable[str],
@@ -916,7 +976,7 @@ def to_version_tuple(ver: Union[float, str], max_split: int = 1):
     if isinstance(ver, float):
         ver = str(ver)
     version_parts = ver.replace('.', '_').split('_', max_split)
-    version_list = [int(part) for part in version_parts]
+    version_list = [int(part) if part.isdigit() else 0 for part in version_parts]
     return tuple(version_list)
 
 

deepfos/options.py

@@ -1,5 +1,6 @@
 import configparser
 import sys
+import threading
 import warnings
 from enum import Enum
 from functools import partial
@@ -114,7 +115,8 @@ def _ensure_discovery_server_is_set(value):
     impl = OPTION.discovery.implementation
     server_opt = {
         'eureka': OPTION.server.eureka,
-        'nacos': OPTION.nacos.server
+        'nacos': OPTION.nacos.server,
+        'k8s': OPTION.k8s.namespace,
     }[impl]
 
     if server_opt is None:
@@ -276,6 +278,10 @@ class _Option(Generic[T_Opt]):
             self.unset_action = _Action.PASS
         self.unset = False
 
+    def _quick_set(self, instance, value):
+        setattr(instance, self._var_name, value)
+        self.unset = False
+
     def __str__(self):
         return self._display_name
 
@@ -366,7 +372,7 @@ class _Category:
         setattr(self, '_name', getattr(other, '_name'))
         for attr, option in self._options.items():
             try:
-                option.__set__(self, getattr(other, attr))
+                option._quick_set(self, getattr(other, attr))  # noqa
             except BaseOptionError:
                 continue
 
@@ -374,9 +380,9 @@ class _Category:
 class _Server(_Category):
     __id__ = 'server_url'
 
-    base = _Option('http://', val_type=str)
+    base = _Option('http://web-gateway', val_type=str)
     app = _Option('http://app-server', val_type=str)
-    account = _Option('http://account-server', val_type=str)
+    account = _Option('http://seepln-account', val_type=str)
     system = _Option('http://system-server', val_type=str)
     space = _Option('http://space-server', val_type=str)
     platform_file = _Option('http://platform-file-server', val_type=str)
@@ -479,16 +485,18 @@ class _Boost(_Category):
 class _ServiceDiscovery(_Category):
     __id__ = 'service_discovery'
 
-    #: 是否使用服务发现功能
+    #: 是否使用服务发现功能（仅影响DynamicAPI）
     enabled = _Option(False, val_type=bool, on_set=_ensure_discovery_server_is_set)
     #: 服务注册发现使用的实现
-    implementation = _Option('eureka', val_type=str, val_choices=('eureka', 'nacos'))
+    implementation = _Option('eureka', val_type=str, val_choices=('eureka', 'nacos', 'k8s'))
     #: 服务注册发现使用的缓存策略
     cache_strategy = _Option(
         'ranked',
         val_type=str,
         val_choices=('ranked', 'roundrobin', 'random')
     )
+    #: 是否完全使用服务发现获取请求地址（包括Root API）
+    take_over = _Option(False, val_type=bool)
 
     def __get__(self, instance, owner) -> '_ServiceDiscovery':
         """defined to help ide"""
@@ -508,6 +516,16 @@ class _Nacos(_Category):
         return super().__get__(instance, owner)
 
 
+class _Kubernets(_Category):
+    """Nacos 相关配置"""
+
+    namespace = _Option(None, val_type=str)
+
+    def __get__(self, instance, owner) -> '_Kubernets':
+        """defined to help ide"""
+        return super().__get__(instance, owner)
+
+
 # -----------------------------------------------------------------------------
 # Options
 class _GlobalOptions:
@@ -520,6 +538,7 @@ class _GlobalOptions:
     discovery = _ServiceDiscovery()
     nacos = _Nacos()
     edgedb = _Edgedb()
+    k8s = _Kubernets()
 
     def load_file(self, filepath):
         parser = configparser.ConfigParser()
@@ -643,19 +662,31 @@ class _OptionCTX:
         boost: _Boost
         discovery: _ServiceDiscovery
         nacos: _Nacos
+        edgedb: _Edgedb
 
         def show_options(self, category=None):
             ...
 
     def __init__(self):
-        self._token = _option_ctx.set(_GlobalOptions())
+        self._create_at_main = threading.current_thread() is threading.main_thread()
+        self._primary_opt = _GlobalOptions()
+        self._token = _option_ctx.set(self._primary_opt)
 
     def create_local(self):
         self._token = _option_ctx.set(_GlobalOptions())
 
     @property
     def _option(self) -> _GlobalOptions:
-        return _option_ctx.get()
+        try:
+            return _option_ctx.get()
+        except LookupError:
+            if (
+                not self._create_at_main
+                and threading.current_thread() is threading.main_thread()
+            ):
+                _option_ctx.set(self._primary_opt.__copy__())
+                return _option_ctx.get()
+            raise
 
     def __getattr__(self, item):
         return getattr(self._option, item)

{deepfos-1.1.60.dist-info → deepfos-1.1.78.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.60
+Version: 1.1.78
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team
@@ -13,15 +13,15 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Requires-Python: >=3.8.3
 Requires-Dist: pandas
-Requires-Dist: requests (==2.25.1)
+Requires-Dist: requests (~=2.26)
 Requires-Dist: pymysql (==1.1.1)
-Requires-Dist: aiohttp (==3.9.0)
-Requires-Dist: pydantic (<2.0.0,>=1.10.0)
-Requires-Dist: cachetools (==4.2.2)
+Requires-Dist: aiohttp (~=3.9.5)
+Requires-Dist: pydantic (!=2.0.0,!=2.0.1,!=2.1.0,>=1.10.0)
+Requires-Dist: cachetools (<6,>=5.0.0)
 Requires-Dist: loguru (==0.5.3)
 Requires-Dist: PyPika (==0.48.8)
-Requires-Dist: aiomysql (==0.2.0)
-Requires-Dist: clickhouse-driver (==0.2.2)
+Requires-Dist: aiomysql (~=0.2.0)
+Requires-Dist: clickhouse-driver (~=0.2.9)
 Requires-Dist: aioredis (==2.0.1)
 Requires-Dist: redis (==4.5.3)
 Requires-Dist: nest-asyncio (==1.5.4)