deepfos 1.1.23__tar.gz → 1.1.26__tar.gz

{deepfos-1.1.23 → deepfos-1.1.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.23
+Version: 1.1.26
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team

{deepfos-1.1.23 → deepfos-1.1.26}/deepfos/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2023-11-15T06:38:21+0000",
+ "date": "2023-12-28T02:24:37+0000",
  "dirty": false,
  "error": null,
- "full-revisionid": "8db1e866488ddeb1ede8a951c6ee035b0eb4e37e",
- "version": "1.1.23"
+ "full-revisionid": "7b2975015e7105274b4054e1136a0dfc22361966",
+ "version": "1.1.26"
 }
 '''  # END VERSION_JSON
 

{deepfos-1.1.23 → deepfos-1.1.26}/deepfos/api/journal_model.py

@@ -30,7 +30,16 @@ class JournalModelData(ChildAPI):
         基于凭证模型的数据批量保存
         """
         return {'body': batch}
-    
+
+    @post('/data/update')
+    def update(self, batch: ModelDataBatchDTO) -> Union[CommonResultDTO, Awaitable[CommonResultDTO]]:
+        """
+        【★】凭证模型-数据-单条更新-只支持头行更新及行插入和删除
+
+        基于凭证模型的数据更新
+        """
+        return {'body': batch}
+
     @post('data/calc/net/amount')
     def calc_net_amount(self, param: JmPostParamVO) -> Union[JmPostResultVO, Awaitable[JmPostResultVO]]:
         """

{deepfos-1.1.23 → deepfos-1.1.26}/deepfos/api/models/financial_model.py

@@ -187,14 +187,16 @@ class TaskExecutionResult(BaseModel):
 
         - **POST** ``/mdx/task/execution`` (Response: 200)
     """
-    #: businessId
+    #: 业务id
     businessId: Optional[str]
-    #: failReason
+    #: 失败原因
     failReason: Optional[str]
-    #: result
+    #: 处理是否正常，true 正常， false 异常
     result: Optional[bool]
-    #: status
+    #: 状态，默认为0(正常返回)，超时为1(超时返回)，判断超时逻辑：result = true & status = 1
     status: Optional[int]
+    #: 计算脚本影响单元格数，模型编辑器入口时返回具体值，其他时候返回null
+    cell: Optional[int]
 
 
 class CubeRightElementDTO(BaseModel):
@@ -1157,6 +1159,8 @@ class ReactSpreadsheetSaveForm(BaseModel):
     wbName: Optional[str]
     #: 电子表格path
     wbPath: Optional[str]
+    #: 是否回调
+    callback: Optional[bool]
 
 
 class SheetColumnDto(BaseModel):

{deepfos-1.1.23 → deepfos-1.1.26}/deepfos/element/finmodel.py

@@ -1,10 +1,13 @@
 import asyncio
 import re
 from enum import Enum
+from itertools import chain
 from typing import (
-    List, Dict, Optional, Union, 
-    Tuple, Iterable, TYPE_CHECKING, Any
+    List, Dict, Optional, Union,
+    Tuple, Iterable, TYPE_CHECKING, Any, Set
 )
+
+import numpy as np
 import pandas as pd
 from pydantic import Field, parse_obj_as
 from loguru import logger
@@ -24,7 +27,7 @@ from deepfos.lib.utils import (
     dict_to_sql, split_dataframe, find_str, concat_url, CIEnumMeta
 )
 from deepfos.lib.constant import (
-    DFLT_DATA_COLUMN, VIEW, VIEW_DICT, 
+    DFLT_DATA_COLUMN, VIEW, VIEW_DICT,
     HIERARCHY, DECIMAL_COL, STRING_COL
 )
 from deepfos.boost import pandas as bp
@@ -41,8 +44,7 @@ from deepfos.api.models.financial_model import (
 from deepfos.api.models.base import BaseModel
 from deepfos.options import OPTION
 from deepfos.lib.decorator import cached_property
-from deepfos.exceptions import MDXExecuteTimeout
-
+from deepfos.exceptions import MDXExecuteTimeout, MDXExecuteFail
 
 __all__ = [
     'AsyncFinancialCube',
@@ -110,6 +112,16 @@ class RoundType(int, Enum, metaclass=CIEnumMeta):
 
 _RE_USE_SECTION = re.compile(r'.*USE\s+\w+;', re.I | re.S)
 
+# 可直接转为mdx member或简单集合函数方法的维度表达式
+# 维度名由字母数字下划线和中划线组成，或者等于#root
+_RE_SIMPLE_EXPR = re.compile(
+    r'(?P<hierarchy>i?(base|descendant|children))'
+    r'\s*\((?P<mbr>(\x23root)|([\w\.\-\[\]]+))\s*,'
+    r'\s*[01]\s*(,\s*(?P<with_parent>[01])\s*)?\)',
+    re.I
+)
+RE_NAME_WITH_PARENT = re.compile(r'\[(.+)]\.\[(.+)]')
+
 
 # -----------------------------------------------------------------------------
 # core classes
@@ -195,6 +207,12 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
 
         return dim_memo
 
+    @cached_property
+    def account_col(self) -> str:
+        for dim in self._meta.cubeDimensionList:
+            if dim.dimensionUsage == 4:
+                return dim.datatableColumn
+
     @cached_property
     def dim_elements(self) -> LazyDict[str, AsyncDimension]:
         """财务Cube的维度元素
@@ -355,7 +373,8 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         pivot_dim: Optional[str] = None,
         validate_expr: bool = True,
         verify_access: bool = False,
-        include_ignored: bool = False
+        include_ignored: bool = False,
+        normalize_view: bool = False,
     ) -> Union[pd.DataFrame, Tuple[pd.DataFrame, Dict[str, str]]]:
         """
         根据维度表达式以及pov获取cube数据
@@ -368,6 +387,7 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
             validate_expr: 是否需要python校验/修改表达式，开启可能会导致额外的接口请求
             verify_access: 是否带权限查询
             include_ignored: 包含多版本实体维时，是否在结果中包含无效数据(即i列为1的数据)
+            normalize_view: 是否把大小写View统一成"View"
 
         .. admonition:: 示例
 
@@ -425,6 +445,13 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
             data = pd.DataFrame(columns=columns)
             data[DFLT_DATA_COLUMN] = data[DFLT_DATA_COLUMN].astype(float)
 
+        if normalize_view:
+            if (view := VIEW.lower()) in data.columns:
+                if VIEW in data.columns:
+                    data[view] = np.where(data[VIEW].isnull(), data[view], data[VIEW])
+                    data = data.drop(columns=[VIEW])
+            data = data.rename(columns=VIEW_DICT)
+
         if pivot_dim is not None:
             pivot_col = self._get_column_from_dim(pivot_dim)
 
@@ -459,6 +486,7 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         need_check: bool = True,
         data_audit: bool = True,
         chunksize: Optional[int] = None,
+        callback: bool = True
     ):
         """
         将DataFrame的数据保存至cube。
@@ -471,6 +499,7 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
             data_audit: 是否需要记录到数据审计
             chunksize: 单次调用保存接口时最大的dataframe行数。
               当data的行数超过此值时，将会分多次进行保存。
+            callback: 是否回调
 
         Note:
             此方法会对落库数据做以下处理：
@@ -511,7 +540,10 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         if missing_dims := required_cols - set(data.columns):
             raise ValueError(
                 f"Cannot save data because following columns are missing: {missing_dims}")
-        return await self._save_impl(data[list(required_cols)], pov, need_check, data_audit, chunksize)
+        return await self._save_impl(
+            data[list(required_cols)],
+            pov, need_check, data_audit, chunksize, callback
+        )
 
     async def save_unpivot(
         self,
@@ -522,6 +554,7 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         data_audit: bool = True,
         chunksize: Optional[int] = None,
         save_nan: bool = False,
+        callback: bool = True
     ):
         """保存有某个维度所有成员在列上的 ``DataFrame``
 
@@ -539,6 +572,7 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
             chunksize: 单次调用保存接口时最大的dataframe行数。
               当data的行数超过此值时，将会分多次进行保存。
             save_nan: 当把数据列成员转换到行上时，data为空的数据是否保存
+            callback: 是否回调
 
         Warnings:
             由于数据完整性等原因，此方法接收的dataframe数据列经常会有一些额外的空值。
@@ -582,7 +616,9 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
                 logger.info("Will not save to cube because dataframe is empty.")
                 return
 
-        return await self._save_impl(data, pov, need_check, data_audit, chunksize)
+        return await self._save_impl(
+            data, pov, need_check, data_audit, chunksize, callback
+        )
 
     async def _save_impl(
         self,
@@ -591,6 +627,7 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         need_check: bool = True,
         data_audit: bool = True,
         chunksize: Optional[int] = None,
+        callback: bool = True
     ):
         # replace NaN to standard None
         data = data.mask(data.isna(), None)
@@ -615,16 +652,144 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
                 needCheck=need_check,
                 dataAuditSwitch=data_audit,
                 entryMode=1,
-                validateDimensionMember=need_check
+                validateDimensionMember=need_check,
+                callback=callback
+            )
+            r = await self.async_api.reactspreadsheet.save(
+                payload.dict(exclude_unset=True)
             )
-            r = await self.async_api.reactspreadsheet.save(payload.dict(exclude_unset=True))
             resp.append(r)
         return resp
 
+    async def delete_with_mdx(
+        self,
+        expression: Union[str, Dict[str, Union[List[str], str]]]
+    ):
+        """通过MDX脚本删除数据
+
+        根据维度表达式删除Cube数据
+
+        Warnings:
+            此方法将根据维度表达式生成对应的MDX脚本并执行MDX的Cleardata
+            对于只有成员和单集合方法的表达式，可以直接转换为MDX的成员集合或集合函数表达式
+            如为复杂表达式(例如包含聚合方法)，则会查询实际对应的成员后，再组成MDX的成员集合
+
+        Args:
+            expression: 维度表达式
+
+        .. admonition:: 示例
+
+            两种调用方式等价：
+
+            .. code-block:: python
+                :emphasize-lines: 3,8
+
+                cube = FinancialCube('example')
+                expr = 'Year{2021;2022}->Entiy{Base(TotalEntity,0)}'
+                r = cube.delete_with_mdx(expr)
+                expr_dict = {
+                    "Year": ['2021', '2022'],
+                    "Entity": "Base(TotalEntity,0)"
+                }
+                r = cube.delete_with_mdx(expr_dict)
+
+        Returns:
+            MDX执行结果
+
+        See Also:
+            :meth:`insert_null` :meth:`delete`
+
+        """
+        if isinstance(expression, dict):
+            expression = dict_to_expr(expression)
+
+        query_dims = []
+        all_cols = set(self.dim_col_map.values())
+        dimexprs = {}
+        cols_appeared = set()
+
+        def normalize_name(name_: str) -> str:
+            if RE_NAME_WITH_PARENT.match(name_):
+                return name_
+            return f'[{name_}]'
+
+        async def query_dim(
+            col_: str,
+            part_: str,
+            dim_name_: str
+        ) -> Tuple[str, Set[str]]:
+            result: List[Dict[str, Any]] = await self.dim_elements[dim_name_].query(
+                part_, fields=['name'], as_model=False
+            )
+            mbrs_ = set()
+            for item in result:
+                if (name := item.get('expectedName')) is not None:
+                    mbrs_.add(f"[{col_}].{normalize_name(name)}")
+                else:
+                    mbrs_.add(f"[{col_}].{normalize_name(item['name'])}")
+            return col_, mbrs_
+
+        for expr in expression.split('->'):
+            dim, body = unpack_expr(expr)
+            col = self._get_column_from_dim(dim)
+
+            for part in body.split(';'):
+                part = part.replace(' ', '')
+                if not part:
+                    continue
+
+                cols_appeared.add(col)
+
+                if is_valid_pov(part):
+                    dimexprs.setdefault(col, set()).add(
+                        f"[{col}].{normalize_name(part)}"
+                    )
+                elif match := _RE_SIMPLE_EXPR.match(part):
+                    mbr = match.group('mbr')
+                    hier = match.group('hierarchy').capitalize()
+                    with_parent = match.group('with_parent')
+
+                    if with_parent == '1':
+                        dimexprs.setdefault(col, set()).add(
+                            f"{hier}([{col}].{normalize_name(mbr)},WITH_PARENT)"
+                        )
+                    else:
+                        dimexprs.setdefault(col, set()).add(
+                            f"{hier}([{col}].{normalize_name(mbr)})"
+                        )
+                else:
+                    dim_name = self.col_dim_map.get(col, col)
+                    query_dims.append(query_dim(col, part, dim_name))
+
+        for col in all_cols - cols_appeared:
+            dimexprs[col] = {f"Base([{col}].[#root])"}
+
+        dim_mbrs = await asyncio.gather(*query_dims)
+
+        for col, mbrs in dim_mbrs:
+            dimexprs.setdefault(col, set())
+            dimexprs[col] = dimexprs[col].union(mbrs)
+
+        # ClearData方法的维度范围优先使用科目维度
+        if account_col := self.account_col:
+            clear_data_expr = dimexprs.pop(account_col)
+            scope_expr = chain(*dimexprs.values())
+        else:
+            expr_list = list(dimexprs.values())
+            clear_data_expr = expr_list[0]
+            scope_expr = chain(*expr_list[1::])
+
+        script = """Scope(%s);\nCleardata(%s);\nEnd Scope;
+        """ % (','.join(scope_expr), ','.join(clear_data_expr))
+
+        return await self.mdx_execution(script)
+
     async def delete(
         self,
         expression: Union[str, Dict[str, Union[List[str], str]]],
-        chunksize: Optional[int] = None
+        chunksize: Optional[int] = None,
+        use_mdx: bool = False,
+        callback: bool = True
     ):
         """删除数据
 
@@ -639,6 +804,8 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
             expression: 维度表达式
             chunksize: 单次调用保存接口时最大的dataframe行数。
               当data的行数超过此值时，将会分多次进行保存。
+            use_mdx: 是否使用MDX脚本实现，默认为否，等效于调用 :meth:`delete_with_mdx`
+            callback: 是否回调
 
         .. admonition:: 示例
 
@@ -660,26 +827,31 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
             删除结果
 
         See Also:
-            :meth:`insert_null`
+            :meth:`insert_null` :meth:`delete_with_mdx`
 
         """
+        if use_mdx:
+            return await self.delete_with_mdx(expression)
+
         if self._meta.autoCalculation:
             if isinstance(expression, str):
                 expression = expr_to_dict(expression)
             expression = {**expression}
-            # expression.pop(VIEW, None)
 
         if isinstance(expression, dict):
             expression = dict_to_expr(expression)
 
         data, pov = await self.query(expression)
         data[DFLT_DATA_COLUMN] = None
-        return await self.save(data, pov, data_audit=True, chunksize=chunksize)
+        return await self.save(
+            data, pov, data_audit=True, chunksize=chunksize, callback=callback
+        )
 
     async def queries(
         self,
         expressions: Iterable[str],
-        drop_duplicates: bool = True
+        drop_duplicates: bool = True,
+        normalize_view: bool = False,
     ) -> pd.DataFrame:
         """查询多个表达式
 
@@ -689,20 +861,23 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         Args:
             expressions: 待查询的维度表达式列表
             drop_duplicates: 是否需要去重
+            normalize_view: 是否把大小写View统一成"View"
 
         Returns:
             查询结果
 
         """
         if isinstance(expressions, str):
-            return await self.query(expressions, compact=False)
+            return await self.query(
+                expressions, compact=False, normalize_view=normalize_view)
 
         expressions = list(expressions)
         if len(expressions) == 1:
-            return await self.query(expressions[0], compact=False)
+            return await self.query(
+                expressions[0], compact=False, normalize_view=normalize_view)
 
         df_list = await asyncio.gather(*(
-            self.query(expr, compact=False)
+            self.query(expr, compact=False, normalize_view=True)
             for expr in expressions
         ))
 
@@ -711,13 +886,12 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
             data = pd.DataFrame(columns=dflt_cols)
             data[DFLT_DATA_COLUMN] = data[DFLT_DATA_COLUMN].astype(float)
         else:
-            if self._meta.autoCalculation:
-                for idx in range(len(df_list)):
-                    df_list[idx] = df_list[idx].rename(columns={VIEW: 'view'})
             data = pd.concat(df_list, sort=False)
             if drop_duplicates:
                 dim_cols = data.columns.difference([DFLT_DATA_COLUMN])
                 data = data.drop_duplicates(dim_cols)
+        if not normalize_view:
+            data = data.rename(columns={VIEW: "view"})
         return data
 
     async def pc_init(
@@ -995,7 +1169,7 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         script: str,
         parameters: Optional[Dict[str, str]] = None,
         precision: Optional[int] = None,
-        timeout: Optional[int] = OPTION.api.timeout * 1000,
+        timeout: Optional[int] = None,
         round_type: Union[RoundType, str] = RoundType.floor,
     ):
         """执行MDX计算语句
@@ -1047,10 +1221,18 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
 
         """
         if _RE_USE_SECTION.match(script.upper()):
-            raise ValueError('MDX语句中发现use section，在FinancialCube中使用时，固定为当前Cube，不支持指定其他Cube')
+            raise ValueError(
+                'MDX语句中发现use section，在FinancialCube中使用时，'
+                '固定为当前Cube，不支持指定其他Cube'
+            )
+
+        if timeout is None:
+            timeout = OPTION.api.timeout * 1000
 
-        business_id = (f"PythonScript_{OPTION.general.task_info.get('script_name', '')}_MDX"
-                       f"-{datetime.datetime.now().strftime('%Y%m%d_%H%M%S_%f')}")
+        business_id = (
+            f"PythonScript_{OPTION.general.task_info.get('script_name', '')}_MDX"
+            f"-{datetime.datetime.now().strftime('%Y%m%d_%H%M%S_%f')}"
+        )
         params = []
 
         if parameters is not None:
@@ -1063,12 +1245,17 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         path = self._path
 
         if path is None:
-            path = await AppAPI(sync=False).folder.get_folder_full(self.element_info.folderId)
+            path = await AppAPI(sync=False).folder.get_folder_full(
+                self.element_info.folderId
+            )
 
         path = path.replace('\\', '/')
 
         params.append(
-            MDXCubeParameter(key=self.element_name, value=concat_url(path, f"{self.element_name}.cub"))
+            MDXCubeParameter(
+                key=self.element_name,
+                value=concat_url(path, f"{self.element_name}.cub")
+            )
         )
 
         res = await self.async_api.mdxtask.execution(
@@ -1085,6 +1272,11 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         if res.status == 1:
             raise MDXExecuteTimeout(f'MDX执行超时，具体响应:\n{res}')
 
+        if res.result is False:
+            raise MDXExecuteFail(
+                f'MDX执行失败，失败原因:\n{res.failReason}'
+            )
+
         return res
 
 
@@ -1095,6 +1287,7 @@ class FinancialCube(AsyncFinancialCube, metaclass=SyncMeta):
         'save',
         'save_unpivot',
         'delete',
+        'delete_with_mdx',
         'pc_init',
         'pc_update',
         'pc_upsert',
@@ -1107,7 +1300,8 @@ class FinancialCube(AsyncFinancialCube, metaclass=SyncMeta):
         def queries(
             self,
             expressions: Iterable[str],
-            drop_duplicates: bool = True
+            drop_duplicates: bool = True,
+            normalize_view: bool = False,
         ) -> pd.DataFrame:
             ...
 
@@ -1119,7 +1313,8 @@ class FinancialCube(AsyncFinancialCube, metaclass=SyncMeta):
             pivot_dim: Optional[str] = None,
             validate_expr: bool = True,
             verify_access: bool = False,
-            include_ignored: bool = False
+            include_ignored: bool = False,
+            normalize_view: bool = False,
         ) -> Union[pd.DataFrame, Tuple[pd.DataFrame, Dict[str, str]]]:
             ...
 
@@ -1131,6 +1326,7 @@ class FinancialCube(AsyncFinancialCube, metaclass=SyncMeta):
             need_check: bool = True,
             data_audit: bool = True,
             chunksize: Optional[int] = None,
+            callback: bool = True
         ):
             ...
 
@@ -1143,13 +1339,22 @@ class FinancialCube(AsyncFinancialCube, metaclass=SyncMeta):
             data_audit: bool = True,
             chunksize: Optional[int] = None,
             save_nan: bool = False,
+            callback: bool = True
         ):
             ...
 
         def delete(
             self,
             expression: Union[str, Dict[str, Union[List[str], str]]],
-            chunksize: Optional[int] = None
+            chunksize: Optional[int] = None,
+            use_mdx: bool = False,
+            callback: bool = True
+        ):
+            ...
+
+        def delete_with_mdx(
+            self,
+            expression: Union[str, Dict[str, Union[List[str], str]]]
         ):
             ...