deepfos 1.1.24__tar.gz → 1.1.25__tar.gz

{deepfos-1.1.24 → deepfos-1.1.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.24
+Version: 1.1.25
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team

{deepfos-1.1.24 → deepfos-1.1.25}/deepfos/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2023-12-12T03:45:14+0000",
+ "date": "2023-12-19T06:52:40+0000",
  "dirty": false,
  "error": null,
- "full-revisionid": "c206dc5e4a3d9eb870bc032b55c17276ea7d9322",
- "version": "1.1.24"
+ "full-revisionid": "0c1304879b006d56f7565a69c2cb418cd6f94968",
+ "version": "1.1.25"
 }
 '''  # END VERSION_JSON
 

{deepfos-1.1.24 → deepfos-1.1.25}/deepfos/api/models/financial_model.py

@@ -1159,6 +1159,8 @@ class ReactSpreadsheetSaveForm(BaseModel):
     wbName: Optional[str]
     #: 电子表格path
     wbPath: Optional[str]
+    #: 是否回调
+    callback: Optional[bool]
 
 
 class SheetColumnDto(BaseModel):

{deepfos-1.1.24 → deepfos-1.1.25}/deepfos/element/finmodel.py

@@ -44,8 +44,7 @@ from deepfos.api.models.financial_model import (
 from deepfos.api.models.base import BaseModel
 from deepfos.options import OPTION
 from deepfos.lib.decorator import cached_property
-from deepfos.exceptions import MDXExecuteTimeout
-
+from deepfos.exceptions import MDXExecuteTimeout, MDXExecuteFail
 
 __all__ = [
     'AsyncFinancialCube',
@@ -487,6 +486,7 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         need_check: bool = True,
         data_audit: bool = True,
         chunksize: Optional[int] = None,
+        callback: bool = True
     ):
         """
         将DataFrame的数据保存至cube。
@@ -499,6 +499,7 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
             data_audit: 是否需要记录到数据审计
             chunksize: 单次调用保存接口时最大的dataframe行数。
               当data的行数超过此值时，将会分多次进行保存。
+            callback: 是否回调
 
         Note:
             此方法会对落库数据做以下处理：
@@ -539,7 +540,10 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         if missing_dims := required_cols - set(data.columns):
             raise ValueError(
                 f"Cannot save data because following columns are missing: {missing_dims}")
-        return await self._save_impl(data[list(required_cols)], pov, need_check, data_audit, chunksize)
+        return await self._save_impl(
+            data[list(required_cols)],
+            pov, need_check, data_audit, chunksize, callback
+        )
 
     async def save_unpivot(
         self,
@@ -550,6 +554,7 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         data_audit: bool = True,
         chunksize: Optional[int] = None,
         save_nan: bool = False,
+        callback: bool = True
     ):
         """保存有某个维度所有成员在列上的 ``DataFrame``
 
@@ -567,6 +572,7 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
             chunksize: 单次调用保存接口时最大的dataframe行数。
               当data的行数超过此值时，将会分多次进行保存。
             save_nan: 当把数据列成员转换到行上时，data为空的数据是否保存
+            callback: 是否回调
 
         Warnings:
             由于数据完整性等原因，此方法接收的dataframe数据列经常会有一些额外的空值。
@@ -610,7 +616,9 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
                 logger.info("Will not save to cube because dataframe is empty.")
                 return
 
-        return await self._save_impl(data, pov, need_check, data_audit, chunksize)
+        return await self._save_impl(
+            data, pov, need_check, data_audit, chunksize, callback
+        )
 
     async def _save_impl(
         self,
@@ -619,6 +627,7 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         need_check: bool = True,
         data_audit: bool = True,
         chunksize: Optional[int] = None,
+        callback: bool = True
     ):
         # replace NaN to standard None
         data = data.mask(data.isna(), None)
@@ -643,18 +652,20 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
                 needCheck=need_check,
                 dataAuditSwitch=data_audit,
                 entryMode=1,
-                validateDimensionMember=need_check
+                validateDimensionMember=need_check,
+                callback=callback
+            )
+            r = await self.async_api.reactspreadsheet.save(
+                payload.dict(exclude_unset=True)
             )
-            r = await self.async_api.reactspreadsheet.save(payload.dict(exclude_unset=True))
             resp.append(r)
         return resp
 
-    async def delete(
+    async def delete_with_mdx(
         self,
-        expression: Union[str, Dict[str, Union[List[str], str]]],
-        chunksize: Optional[int] = None
+        expression: Union[str, Dict[str, Union[List[str], str]]]
     ):
-        """删除数据
+        """通过MDX脚本删除数据
 
         根据维度表达式删除Cube数据
 
@@ -665,7 +676,6 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
 
         Args:
             expression: 维度表达式
-            chunksize: 目前使用MDX实现，已弃用(弃用版本: 1.1.24)，仅兼容旧版本保留
 
         .. admonition:: 示例
 
@@ -676,25 +686,20 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
 
                 cube = FinancialCube('example')
                 expr = 'Year{2021;2022}->Entiy{Base(TotalEntity,0)}'
-                r = cube.delete(expr)
+                r = cube.delete_with_mdx(expr)
                 expr_dict = {
                     "Year": ['2021', '2022'],
                     "Entity": "Base(TotalEntity,0)"
                 }
-                r = cube.delete(expr_dict)
+                r = cube.delete_with_mdx(expr_dict)
 
         Returns:
-            删除结果
+            MDX执行结果
 
         See Also:
-            :meth:`insert_null`
+            :meth:`insert_null` :meth:`delete`
 
         """
-        if chunksize:
-            logger.warning(
-                'Parameter \'chunksize\' is deprecated since 1.1.24'
-            )
-
         if isinstance(expression, dict):
             expression = dict_to_expr(expression)
 
@@ -779,6 +784,69 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
 
         return await self.mdx_execution(script)
 
+    async def delete(
+        self,
+        expression: Union[str, Dict[str, Union[List[str], str]]],
+        chunksize: Optional[int] = None,
+        use_mdx: bool = False,
+        callback: bool = True
+    ):
+        """删除数据
+
+        根据维度表达式删除Cube数据。
+
+        Warnings:
+            此方法首先查询数据，并且替换为null再调用保存接口。
+            因此如果要删除的数据量大，可能导致内存不足等问题。
+            如果不需要数据审计功能，请使用 :meth:`insert_null`
+
+        Args:
+            expression: 维度表达式
+            chunksize: 单次调用保存接口时最大的dataframe行数。
+              当data的行数超过此值时，将会分多次进行保存。
+            use_mdx: 是否使用MDX脚本实现，默认为否，等效于调用 :meth:`delete_with_mdx`
+            callback: 是否回调
+
+        .. admonition:: 示例
+
+            两种调用方式等价：
+
+            .. code-block:: python
+                :emphasize-lines: 3,8
+
+                cube = FinancialCube('example')
+                expr = 'Year{2021;2022}->Entiy{Base(TotalEntity,0)}'
+                r = cube.delete(expr)
+                expr_dict = {
+                    "Year": ['2021', '2022'],
+                    "Entity": "Base(TotalEntity,0)"
+                }
+                r = cube.delete(expr_dict)
+
+        Returns:
+            删除结果
+
+        See Also:
+            :meth:`insert_null` :meth:`delete_with_mdx`
+
+        """
+        if use_mdx:
+            return await self.delete_with_mdx(expression)
+
+        if self._meta.autoCalculation:
+            if isinstance(expression, str):
+                expression = expr_to_dict(expression)
+            expression = {**expression}
+
+        if isinstance(expression, dict):
+            expression = dict_to_expr(expression)
+
+        data, pov = await self.query(expression)
+        data[DFLT_DATA_COLUMN] = None
+        return await self.save(
+            data, pov, data_audit=True, chunksize=chunksize, callback=callback
+        )
+
     async def queries(
         self,
         expressions: Iterable[str],
@@ -1204,6 +1272,11 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         if res.status == 1:
             raise MDXExecuteTimeout(f'MDX执行超时，具体响应:\n{res}')
 
+        if res.result is False:
+            raise MDXExecuteFail(
+                f'MDX执行失败，失败原因:\n{res.failReason}'
+            )
+
         return res
 
 
@@ -1214,6 +1287,7 @@ class FinancialCube(AsyncFinancialCube, metaclass=SyncMeta):
         'save',
         'save_unpivot',
         'delete',
+        'delete_with_mdx',
         'pc_init',
         'pc_update',
         'pc_upsert',
@@ -1252,6 +1326,7 @@ class FinancialCube(AsyncFinancialCube, metaclass=SyncMeta):
             need_check: bool = True,
             data_audit: bool = True,
             chunksize: Optional[int] = None,
+            callback: bool = True
         ):
             ...
 
@@ -1264,13 +1339,22 @@ class FinancialCube(AsyncFinancialCube, metaclass=SyncMeta):
             data_audit: bool = True,
             chunksize: Optional[int] = None,
             save_nan: bool = False,
+            callback: bool = True
         ):
             ...
 
         def delete(
             self,
             expression: Union[str, Dict[str, Union[List[str], str]]],
-            chunksize: Optional[int] = None
+            chunksize: Optional[int] = None,
+            use_mdx: bool = False,
+            callback: bool = True
+        ):
+            ...
+
+        def delete_with_mdx(
+            self,
+            expression: Union[str, Dict[str, Union[List[str], str]]]
         ):
             ...
 

{deepfos-1.1.24 → deepfos-1.1.25}/deepfos/exceptions/__init__.py

@@ -129,6 +129,10 @@ class MDXExecuteTimeout(DeepfosBaseException):
     """执行MDX任务超时"""
 
 
+class MDXExecuteFail(DeepfosBaseException):
+    """执行MDX任务失败"""
+
+
 # -----------------------------------------------------------------------------
 # DeepModel Error
 class DeepModelError(DeepfosBaseException):

{deepfos-1.1.24 → deepfos-1.1.25}/deepfos/lib/asynchronous.py

@@ -1,7 +1,10 @@
+from __future__ import annotations
+
 import asyncio
 import itertools
 from asyncio import SelectorEventLoop
 import asyncio.events as events
+from asyncio import Future as AioFuture
 import sys
 from contextlib import contextmanager
 from heapq import heappop
@@ -14,7 +17,7 @@ import atexit
 from typing import (
     Optional, Tuple, Union, TYPE_CHECKING, List,
     Callable, NamedTuple, Coroutine, Any, Dict,
-    Generic, Iterable, TypeVar, Awaitable, overload
+    Generic, Iterable, TypeVar, Awaitable, overload,
 )
 # noinspection PyProtectedMember
 from concurrent.futures._base import (
@@ -269,11 +272,11 @@ class LinkedList(Generic[LinkListItem]):
             self.remove(node)
 
 
-class _AbstractTask(Generic[FutureT]):
+class _AbstractTask(Generic[ReturnT]):
     def __init__(
         self,
-        future: FutureT,
-        coro: Coroutine,
+        future: Union[Future[ReturnT], AioFuture[ReturnT]],
+        coro: Coroutine[ReturnT, Any, ReturnT],
         ensure_completed: bool = False,
         die_in_peace: bool = False
     ):
@@ -282,10 +285,10 @@ class _AbstractTask(Generic[FutureT]):
         self.ensure_completed = ensure_completed
         self.die_in_peace = die_in_peace
 
-    def result(self, timeout: Optional[float] = None):  # pragma: no cover
+    def result(self, timeout: Optional[float] = None) -> ReturnT:  # pragma: no cover
         raise NotImplementedError
 
-    def done(self):
+    def done(self) -> bool:
         return self.future.done()
 
     def add_done_callback(self, callback):
@@ -305,26 +308,40 @@ class _AbstractTask(Generic[FutureT]):
         )
 
 
-class AsyncIOTask(_AbstractTask[asyncio.Future]):
-    def __init__(self, future, coro, ensure):
-        super().__init__(future, coro, ensure)
+class AsyncIOTask(_AbstractTask):
+    def __init__(
+        self,
+        future: AioFuture[ReturnT],
+        coro: Coroutine[ReturnT, Any, ReturnT],
+        ensure_completed: bool = False,
+    ):
+        super().__init__(future, coro, ensure_completed)
         self.r = UNSET
+        self.exc = None
         self.future.add_done_callback(self.set_task_done)
 
-    def set_task_done(self, future: asyncio.Future):
-        self.r = future.result()
+    def set_task_done(self, future: AioFuture):
+        if exc := future.exception():
+            self.exc = exc
+        else:
+            self.r = future.result()
 
     async def _wait_for_result(self, timeout=None):
         return await asyncio.wait_for(self.future, timeout)
 
-    def result(self, timeout: Optional[float] = None):
+    def result(self, timeout: Optional[float] = None) -> ReturnT:
+        if self.exc is not None:
+            raise self.exc
         if self.r is UNSET:
             evloop.run(self._wait_for_result(timeout))
         return self.r
 
 
-class ThreadTask(_AbstractTask[Future]):
-    def result(self, timeout: Optional[float] = None):
+class ThreadTask(_AbstractTask):
+    if TYPE_CHECKING:
+        future: Future
+
+    def result(self, timeout: Optional[float] = None) -> ReturnT:
         if (
             evloop.in_same_thread()
             and not (
@@ -419,13 +436,17 @@ class EventLoop:
     def set_loop(self, loop: asyncio.BaseEventLoop):
         self.__loop = loop
 
-    def run(self, coro):
+    def run(self, coro: Coroutine[ReturnT, Any, ReturnT]) -> ReturnT:
         if self.in_same_thread():
             return self.loop.run_until_complete(coro)
         else:
             return self.create_task(coro).result()
 
-    def create_task(self, coro, ensure_completed=False) -> _AbstractTask:
+    def create_task(
+        self,
+        coro: Coroutine[ReturnT, Any, ReturnT],
+        ensure_completed=False
+    ) -> _AbstractTask[ReturnT]:
         task: _AbstractTask
         if self.in_same_thread():
             task = AsyncIOTask(
@@ -446,7 +467,12 @@ class EventLoop:
 
     def _report_task_done(self, node: Node):
         def cb(fut):
-            logger.debug(f"Task {node.value} finished.")
+            def get_hint():
+                if exc := fut.exception():
+                    return f'failed with exception: {exc!r}'
+                else:
+                    return f'finsihed with result: {fut.result()}'
+            logger.opt(lazy=True).debug(f"Task {node.value} {{}}", get_hint)
             self._tasks.remove(node)
         return cb
 
@@ -539,7 +565,7 @@ class EventLoop:
             periodic_task.task.cancel()
             if periodic_task.on_stop is not None:
                 self.run(periodic_task.on_stop())
-        except Exception:  # pragma: no cover
+        except Exception:  # pragma: no cover  # noqa
             logger.exception("Exception occurs when cancel task.")
 
 
@@ -599,16 +625,16 @@ class _TaskProp:
 class future_property(Generic[ReturnT]):
     def __init__(
         self, 
-        func: Optional[Callable[[Any], Awaitable[ReturnT]]] = None,
-        on_demand=False
+        func: Optional[Callable[[Any], Coroutine[ReturnT, Any, ReturnT]]] = None,
+        on_demand: bool = False
     ):
         self.func = func
         self.on_demand = on_demand
         if func is not None:
             self.__doc__ = func.__doc__
-        self._tasks: Dict[Any, _TaskProp] = {}
+        self._tasks: Dict[int, _TaskProp] = {}
 
-    def __call__(self, func: Callable[[Any], Awaitable[ReturnT]]):
+    def __call__(self, func: Callable[[Any], Coroutine[ReturnT, Any, ReturnT]]):
         self.func = func
         self.__doc__ = func.__doc__
         return self
@@ -778,7 +804,7 @@ class DeepFOSIO:
         finally:
             try:
                 await self.apply_handler(self.on_shutdown)
-            except Exception:  # pragma: no cover
+            except Exception:  # pragma: no cover  # noqa
                 pass
 
 

{deepfos-1.1.24 → deepfos-1.1.25}/deepfos/lib/serutils.py

@@ -107,39 +107,23 @@ def _set(o):
     return [serialize(el) for el in o]
 
 
-@serialize.register(uuid.UUID)
-def _stringify(o):
-    return str(o)
-
-
 @serialize.register(int)
 @serialize.register(float)
 @serialize.register(str)
 @serialize.register(bytes)
 @serialize.register(bool)
 @serialize.register(type(None))
+@serialize.register(uuid.UUID)
 @serialize.register(decimal.Decimal)
 @serialize.register(datetime.timedelta)
+@serialize.register(datetime.date)
+@serialize.register(datetime.datetime)
+@serialize.register(datetime.time)
 @serialize.register(edgedb.RelativeDuration)
 def _scalar(o):
     return o
 
 
-@serialize.register
-def _datetime(o: datetime.datetime):
-    return o.isoformat()
-
-
-@serialize.register
-def _date(o: datetime.date):
-    return o.isoformat()
-
-
-@serialize.register
-def _time(o: datetime.time):
-    return o.isoformat()
-
-
 @serialize.register
 def _enum(o: edgedb.EnumValue):
     return str(o)

{deepfos-1.1.24 → deepfos-1.1.25}/deepfos.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.24
+Version: 1.1.25
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team