deepfos 1.1.55__tar.gz → 1.1.56__tar.gz

{deepfos-1.1.55 → deepfos-1.1.56}/CHANGELOG.md

@@ -1,3 +1,10 @@
+## [1.1.56] - 2025-04-27
+
+### 新增
+
+* 财务模型支持complement_save_unpivot
+
+
 ## [1.1.55] - 2025-04-25
 
 ### 新增

{deepfos-1.1.55 → deepfos-1.1.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.55
+Version: 1.1.56
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team

{deepfos-1.1.55 → deepfos-1.1.56}/deepfos/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-04-27T01:54:31+0000",
+ "date": "2025-04-28T08:53:07+0000",
  "dirty": false,
  "error": null,
- "full-revisionid": "8a98c948a65577c31df7cff7c2a512e8d71de104",
- "version": "1.1.55"
+ "full-revisionid": "65654605edcd2831ed325b40b62f71c933313017",
+ "version": "1.1.56"
 }
 '''  # END VERSION_JSON
 

{deepfos-1.1.55 → deepfos-1.1.56}/deepfos/element/deepmodel.py

@@ -774,8 +774,12 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
     """DeepModel元素
 
     Args:
-        direct_access: 是否使用直连模式
+        direct_access: 是否使用直连模式，默认为True
+            会结合OPTION.edgedb.dsn是否有值决定是否使用直连模式，如无值，则仍为非直连模式
+            直连模式下，会使用edgedb-python库直连edgedb server，
+            否则会使用DeepModel组件API进行操作
         pg_dsn: PG连接信息
+
     """
     __mangle_docs__ = False
 
@@ -783,7 +787,9 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         self._txn_ = ContextVar('QLTXN')
         self.appmodule = f"app{OPTION.api.header['app']}"
         self.spacemodule = f"space{OPTION.api.header['space']}"
-        self.direct_access = direct_access
+        self.direct_access = direct_access and bool(OPTION.edgedb.dsn)
+        if not self.direct_access:
+            logger.debug('当前DeepModel为非直连模式')
         self.alias = AliasGenerator()
         self.pg_dsn = pg_dsn
         self._globals = {}

{deepfos-1.1.55 → deepfos-1.1.56}/deepfos/element/finmodel.py

@@ -487,10 +487,10 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         pov: Optional[Union[str, Dict[str, str]]] = None,
         data_column: str = DFLT_DATA_COLUMN,
         comment_column: str = DFLT_COMMENT_COLUMN,
-    ) -> pd.DataFrame:
+    ) -> Tuple[pd.DataFrame, Dict[str, str]]:
         if data.empty:
             logger.info("Will not save to cube because dataframe is empty.")
-            return pd.DataFrame()
+            return pd.DataFrame(), {}
 
         if data_column not in data.columns:
             raise ValueError(f"Missing data column: {data_column}.")
@@ -537,7 +537,7 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
             data = pd.concat([data, cmt_data])
             required_cols.add(COLUMN_USAGE_FIELD)
 
-        return data[list(required_cols)]
+        return data[list(required_cols)], pov
 
     async def save(
         self,
@@ -580,11 +580,34 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
             :meth:`save_unpivot`
 
         """
-        data = self._build_dataframe_for_save(data, pov, data_column, comment_column)
+        data, pov = self._build_dataframe_for_save(data, pov, data_column, comment_column)
+        if data.empty:
+            return
+
         return await self._save_impl(
             data, pov, need_check, data_audit, chunksize, callback, auth_mode
         )
 
+    def _complement(
+        self,
+        data: pd.DataFrame,
+        expression: str,
+        default_hierarchy: str = 'Base',
+        pov: Dict[str, str] = None,
+    ):
+        if isinstance(expression, dict):
+            expression = dict_to_expr(expression)
+        expr, pov = self._split_expr(expression, pov or {}, default_hierarchy)
+        full_expr = {**expr_to_dict(expr), **pov}
+        folders = {col: dim.folderId for col, dim in self.dimensions.items()}
+        data_comp = complete_cartesian_product(
+            full_expr,
+            data,
+            folder_ids=folders,
+            col_dim_map=self.col_dim_map,
+        )
+        return data_comp
+
     async def complement_save(
         self,
         data: pd.DataFrame,
@@ -619,22 +642,42 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
             **kwargs: 其他可传给:meth:`save`的参数
 
         """
-        data = self._build_dataframe_for_save(data, pov, data_column, comment_column)
+        data, pov = self._build_dataframe_for_save(data, pov, data_column, comment_column)
         if data.empty:
             return
 
-        if isinstance(expression, dict):
-            expression = dict_to_expr(expression)
-        expr, pov = self._split_expr(expression, pov or {}, default_hierarchy)
-        full_expr = {**expr_to_dict(expr), **pov}
+        data_comp = self._complement(data, expression, default_hierarchy, pov)
+        return await self._save_impl(data_comp, pov, **kwargs)
 
-        folders = {col: dim.folderId for col, dim in self.dimensions.items()}
-        data_comp = complete_cartesian_product(
-            full_expr,
-            data,
-            folder_ids=folders,
+    def _build_dataframe_for_save_unpivot(
+        self,
+        data: pd.DataFrame,
+        unpivot_dim: str,
+        pov: Optional[Union[str, Dict[str, str]]] = None,
+        save_nan: bool = False,
+    ) -> Tuple[pd.DataFrame, Dict[str, str]]:
+        if data.empty:
+            logger.info("Will not save to cube because dataframe is empty.")
+            return pd.DataFrame(), {}
+
+        data = data.rename(columns=self._maybe_get_column_from_dim)
+        dim = self._get_column_from_dim(unpivot_dim)
+        pov = self._resolve_pov_as_dict(pov)
+        data_cols = set(data.columns)
+        unpivot_cols = data_cols.difference(pov.keys(), self.dim_col_map.values())
+        if self._meta.autoCalculation:
+            unpivot_cols.discard(VIEW)
+        id_cols = data_cols - unpivot_cols
+        data = data.melt(
+            id_vars=id_cols, value_vars=unpivot_cols,
+            var_name=dim, value_name=DFLT_DATA_COLUMN
         )
-        return await self._save_impl(data_comp, **kwargs)
+        if not save_nan:
+            data = data.dropna()
+            if data.empty:
+                logger.info("Will not save to cube because dataframe is empty.")
+                return pd.DataFrame(), {}
+        return data, pov
 
     async def save_unpivot(
         self,
@@ -682,34 +725,62 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
             | :meth:`save`
 
         """
+        data, pov = self._build_dataframe_for_save_unpivot(
+            data, unpivot_dim, pov, save_nan
+        )
         if data.empty:
-            logger.info("Will not save to cube because dataframe is empty.")
             return
 
-        data = data.rename(columns=self._maybe_get_column_from_dim)
-        dim = self._get_column_from_dim(unpivot_dim)
-        pov = self._resolve_pov_as_dict(pov)
-        data_cols = set(data.columns)
-        unpivot_cols = data_cols.difference(pov.keys(), self.dim_col_map.values())
+        return await self._save_impl(
+            data, pov, need_check, data_audit, chunksize, callback
+        )
 
-        if self._meta.autoCalculation:
-            unpivot_cols.discard(VIEW)
+    async def complement_save_unpivot(
+        self,
+        data: pd.DataFrame,
+        unpivot_dim: str,
+        expression: Union[str, Dict[str, Union[List[str], str]]],
+        default_hierarchy: str = "Base",
+        pov: Optional[Union[str, Dict[str, str]]] = None,
+        save_nan: bool = False,
+        **kwargs
+    ):
+        """覆盖指定维度范围并保有某个维度所有成员在列上的 ``DataFrame``
 
-        id_cols = data_cols - unpivot_cols
-        data = data.melt(
-            id_vars=id_cols, value_vars=unpivot_cols,
-            var_name=dim, value_name=DFLT_DATA_COLUMN
-        )
+        相比于:meth:`save_unpivot`，在保存前，会将`data`按照`expression`补全笛卡尔积。
+        并且不在`data`范围的数据以`None`填充
 
-        if not save_nan:
-            data = data.dropna()
-            if data.empty:
-                logger.info("Will not save to cube because dataframe is empty.")
-                return
+        Note:
+            逻辑上等价于两次调用
 
-        return await self._save_impl(
-            data, pov, need_check, data_audit, chunksize, callback
+            .. code-block:: python
+
+                cube.delete(expression)
+                cube.save_unpivot(data, unpivot_dim, **kwargs)
+
+        Args:
+            data: 需要保存的数据
+            unpivot_dim: 成员在列上的维度
+            expression: 需要覆盖的范围（维度表达式）
+            default_hierarchy: 单expression中没指定对应维度时，默认取的层级函数，
+                即填充为 `default_hierarchy(#root,0)`
+            pov: Point Of View，维度表达式或者KV键值对格式。
+            need_check: 是否需要java接口校验脏数据
+            data_audit: 是否需要记录到数据审计
+            chunksize: 单次调用保存接口时最大的dataframe行数。
+              当data的行数超过此值时，将会分多次进行保存。
+            save_nan: 当把数据列成员转换到行上时，data为空的数据是否保存
+            callback: 是否回调
+
+        """
+        data, pov = self._build_dataframe_for_save_unpivot(
+            data, unpivot_dim, pov, save_nan
         )
+        if data.empty:
+            return
+
+        data_comp = self._complement(data, expression, default_hierarchy, pov)
+        return await self._save_impl(data_comp, pov, **kwargs)
 
     async def _save_impl(
         self,
@@ -722,7 +793,9 @@ class AsyncFinancialCube(ElementBase[FinancialModelAPI]):
         auth_mode: Literal[0, 1, 2, 3] = 0,
     ):
         # replace NaN to standard None
-        data = data.mask(data.isna(), None)
+        # NB: replace twice in case of infer None to nan happened in 2.x pandas
+        data = data.replace({None: np.nan})
+        data = data.replace({np.nan: None})
         # ensure view is capitalized
         if self._meta.autoCalculation:
             data = data.rename(columns=VIEW_DICT)
@@ -1381,6 +1454,7 @@ class FinancialCube(AsyncFinancialCube, metaclass=SyncMeta):
         'save',
         'complement_save',
         'save_unpivot',
+        'complement_save_unpivot',
         'delete',
         'delete_with_mdx',
         'pc_init',
@@ -1388,7 +1462,7 @@ class FinancialCube(AsyncFinancialCube, metaclass=SyncMeta):
         'pc_upsert',
         'insert_null',
         'copy_calculate',
-        'mdx_execution'
+        'mdx_execution',
     )
 
     if TYPE_CHECKING:   # pragma: no cover
@@ -1451,6 +1525,18 @@ class FinancialCube(AsyncFinancialCube, metaclass=SyncMeta):
         ):
             ...
 
+        def complement_save_unpivot(
+            self,
+            data: pd.DataFrame,
+            unpivot_dim: str,
+            expression: Union[str, Dict[str, Union[List[str], str]]],
+            default_hierarchy: str = "Base",
+            pov: Optional[Union[str, Dict[str, str]]] = None,
+            save_nan: bool = False,
+            **kwargs
+        ):
+            ...
+
         def delete(
             self,
             expression: Union[str, Dict[str, Union[List[str], str]]],

{deepfos-1.1.55 → deepfos-1.1.56}/deepfos/lib/sysutils.py

@@ -105,6 +105,7 @@ def complete_cartesian_product(
     df: pd.DataFrame = None,
     paths: Union[str, Dict[str, str]] = None,
     folder_ids: Union[str, Dict[str, str]] = None,
+    col_dim_map: Dict[str, str] = None,
 ) -> pd.DataFrame:
     """
     构造完整的维度成员笛卡尔积
@@ -115,6 +116,7 @@ def complete_cartesian_product(
         paths: fix中维度的path，如果所有维度的目录相同，传同一个path，否则传字典，key为维度名，value为path。
           如果不传，则自动寻找维度对应的path。
         folder_ids: 类似paths, 但值是folder_id
+        col_dim_map: data中的列名与实际维度名的映射关系，默认data中的列名与维度名相同
 
     Returns:
         维度成员笛卡尔积的DataFrame
@@ -165,29 +167,33 @@ def complete_cartesian_product(
     else:
         loc_key = "path"
         loc_getter = lambda _: None
+    if col_dim_map is None:
+        col_dim_map = {}
 
     # 遍历fix，如果fix的值为str，则认为是维度表达式，将表达式转换为成员list
     mbrs = {}
     futures = []
 
-    for dim, exp in fix.items():
+    for col, exp in fix.items():
         if isinstance(exp, str):
             if "(" not in exp:
-                mbrs[dim] = exp.split(';')
+                mbrs[col] = exp.split(';')
             else:
+                dim = col_dim_map.get(col, col)
                 loc = loc_getter(dim)
                 future = evloop.apply(AsyncDimension(element_name=dim, **{loc_key: loc}).query(
                     expression=exp, fields=['name'], as_model=False
                 ))
 
-                futures.append((dim, future))
+                futures.append((col, future))
         else:
             if not isinstance(exp, list):
                 raise TypeError('fix参数的value只能为维度表达式(str)或维度成员(list)')
-            mbrs[dim] = exp
+            mbrs[col] = exp
 
-    for dim, future in futures:
-        mbrs[dim] = [item['name'] for item in future.result()]
+    for col, future in futures:
+        dim_mbrs = future.result()
+        mbrs[col] = [item['name'] for item in dim_mbrs]
 
     if df is None:
         df = pd.DataFrame(columns=list(fix.keys()))

{deepfos-1.1.55 → deepfos-1.1.56}/deepfos.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.55
+Version: 1.1.56
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team