deepfos 1.1.60__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. deepfos/__init__.py +6 -0
  2. deepfos/_version.py +21 -0
  3. deepfos/algo/__init__.py +0 -0
  4. deepfos/algo/graph.py +171 -0
  5. deepfos/algo/segtree.py +31 -0
  6. deepfos/api/V1_1/__init__.py +0 -0
  7. deepfos/api/V1_1/business_model.py +119 -0
  8. deepfos/api/V1_1/dimension.py +599 -0
  9. deepfos/api/V1_1/models/__init__.py +0 -0
  10. deepfos/api/V1_1/models/business_model.py +1033 -0
  11. deepfos/api/V1_1/models/dimension.py +2768 -0
  12. deepfos/api/V1_2/__init__.py +0 -0
  13. deepfos/api/V1_2/dimension.py +285 -0
  14. deepfos/api/V1_2/models/__init__.py +0 -0
  15. deepfos/api/V1_2/models/dimension.py +2923 -0
  16. deepfos/api/__init__.py +0 -0
  17. deepfos/api/account.py +167 -0
  18. deepfos/api/accounting_engines.py +147 -0
  19. deepfos/api/app.py +626 -0
  20. deepfos/api/approval_process.py +198 -0
  21. deepfos/api/base.py +983 -0
  22. deepfos/api/business_model.py +160 -0
  23. deepfos/api/consolidation.py +129 -0
  24. deepfos/api/consolidation_process.py +106 -0
  25. deepfos/api/datatable.py +341 -0
  26. deepfos/api/deep_pipeline.py +61 -0
  27. deepfos/api/deepconnector.py +36 -0
  28. deepfos/api/deepfos_task.py +92 -0
  29. deepfos/api/deepmodel.py +188 -0
  30. deepfos/api/dimension.py +486 -0
  31. deepfos/api/financial_model.py +319 -0
  32. deepfos/api/journal_model.py +119 -0
  33. deepfos/api/journal_template.py +132 -0
  34. deepfos/api/memory_financial_model.py +98 -0
  35. deepfos/api/models/__init__.py +3 -0
  36. deepfos/api/models/account.py +483 -0
  37. deepfos/api/models/accounting_engines.py +756 -0
  38. deepfos/api/models/app.py +1338 -0
  39. deepfos/api/models/approval_process.py +1043 -0
  40. deepfos/api/models/base.py +234 -0
  41. deepfos/api/models/business_model.py +805 -0
  42. deepfos/api/models/consolidation.py +711 -0
  43. deepfos/api/models/consolidation_process.py +248 -0
  44. deepfos/api/models/datatable_mysql.py +427 -0
  45. deepfos/api/models/deep_pipeline.py +55 -0
  46. deepfos/api/models/deepconnector.py +28 -0
  47. deepfos/api/models/deepfos_task.py +386 -0
  48. deepfos/api/models/deepmodel.py +308 -0
  49. deepfos/api/models/dimension.py +1576 -0
  50. deepfos/api/models/financial_model.py +1796 -0
  51. deepfos/api/models/journal_model.py +341 -0
  52. deepfos/api/models/journal_template.py +854 -0
  53. deepfos/api/models/memory_financial_model.py +478 -0
  54. deepfos/api/models/platform.py +178 -0
  55. deepfos/api/models/python.py +221 -0
  56. deepfos/api/models/reconciliation_engine.py +411 -0
  57. deepfos/api/models/reconciliation_report.py +161 -0
  58. deepfos/api/models/role_strategy.py +884 -0
  59. deepfos/api/models/smartlist.py +237 -0
  60. deepfos/api/models/space.py +1137 -0
  61. deepfos/api/models/system.py +1065 -0
  62. deepfos/api/models/variable.py +463 -0
  63. deepfos/api/models/workflow.py +946 -0
  64. deepfos/api/platform.py +199 -0
  65. deepfos/api/python.py +90 -0
  66. deepfos/api/reconciliation_engine.py +181 -0
  67. deepfos/api/reconciliation_report.py +64 -0
  68. deepfos/api/role_strategy.py +234 -0
  69. deepfos/api/smartlist.py +69 -0
  70. deepfos/api/space.py +582 -0
  71. deepfos/api/system.py +372 -0
  72. deepfos/api/variable.py +154 -0
  73. deepfos/api/workflow.py +264 -0
  74. deepfos/boost/__init__.py +6 -0
  75. deepfos/boost/py_jstream.py +89 -0
  76. deepfos/boost/py_pandas.py +20 -0
  77. deepfos/cache.py +121 -0
  78. deepfos/config.py +6 -0
  79. deepfos/core/__init__.py +27 -0
  80. deepfos/core/cube/__init__.py +10 -0
  81. deepfos/core/cube/_base.py +462 -0
  82. deepfos/core/cube/constants.py +21 -0
  83. deepfos/core/cube/cube.py +408 -0
  84. deepfos/core/cube/formula.py +707 -0
  85. deepfos/core/cube/syscube.py +532 -0
  86. deepfos/core/cube/typing.py +7 -0
  87. deepfos/core/cube/utils.py +238 -0
  88. deepfos/core/dimension/__init__.py +11 -0
  89. deepfos/core/dimension/_base.py +506 -0
  90. deepfos/core/dimension/dimcreator.py +184 -0
  91. deepfos/core/dimension/dimension.py +472 -0
  92. deepfos/core/dimension/dimexpr.py +271 -0
  93. deepfos/core/dimension/dimmember.py +155 -0
  94. deepfos/core/dimension/eledimension.py +22 -0
  95. deepfos/core/dimension/filters.py +99 -0
  96. deepfos/core/dimension/sysdimension.py +168 -0
  97. deepfos/core/logictable/__init__.py +5 -0
  98. deepfos/core/logictable/_cache.py +141 -0
  99. deepfos/core/logictable/_operator.py +663 -0
  100. deepfos/core/logictable/nodemixin.py +673 -0
  101. deepfos/core/logictable/sqlcondition.py +609 -0
  102. deepfos/core/logictable/tablemodel.py +497 -0
  103. deepfos/db/__init__.py +36 -0
  104. deepfos/db/cipher.py +660 -0
  105. deepfos/db/clickhouse.py +191 -0
  106. deepfos/db/connector.py +195 -0
  107. deepfos/db/daclickhouse.py +171 -0
  108. deepfos/db/dameng.py +101 -0
  109. deepfos/db/damysql.py +189 -0
  110. deepfos/db/dbkits.py +358 -0
  111. deepfos/db/deepengine.py +99 -0
  112. deepfos/db/deepmodel.py +82 -0
  113. deepfos/db/deepmodel_kingbase.py +83 -0
  114. deepfos/db/edb.py +214 -0
  115. deepfos/db/gauss.py +83 -0
  116. deepfos/db/kingbase.py +83 -0
  117. deepfos/db/mysql.py +184 -0
  118. deepfos/db/oracle.py +131 -0
  119. deepfos/db/postgresql.py +192 -0
  120. deepfos/db/sqlserver.py +99 -0
  121. deepfos/db/utils.py +135 -0
  122. deepfos/element/__init__.py +89 -0
  123. deepfos/element/accounting.py +348 -0
  124. deepfos/element/apvlprocess.py +215 -0
  125. deepfos/element/base.py +398 -0
  126. deepfos/element/bizmodel.py +1269 -0
  127. deepfos/element/datatable.py +2467 -0
  128. deepfos/element/deep_pipeline.py +186 -0
  129. deepfos/element/deepconnector.py +59 -0
  130. deepfos/element/deepmodel.py +1806 -0
  131. deepfos/element/dimension.py +1254 -0
  132. deepfos/element/fact_table.py +427 -0
  133. deepfos/element/finmodel.py +1485 -0
  134. deepfos/element/journal.py +840 -0
  135. deepfos/element/journal_template.py +943 -0
  136. deepfos/element/pyscript.py +412 -0
  137. deepfos/element/reconciliation.py +553 -0
  138. deepfos/element/rolestrategy.py +243 -0
  139. deepfos/element/smartlist.py +457 -0
  140. deepfos/element/variable.py +756 -0
  141. deepfos/element/workflow.py +560 -0
  142. deepfos/exceptions/__init__.py +239 -0
  143. deepfos/exceptions/hook.py +86 -0
  144. deepfos/lazy.py +104 -0
  145. deepfos/lazy_import.py +84 -0
  146. deepfos/lib/__init__.py +0 -0
  147. deepfos/lib/_javaobj.py +366 -0
  148. deepfos/lib/asynchronous.py +879 -0
  149. deepfos/lib/concurrency.py +107 -0
  150. deepfos/lib/constant.py +39 -0
  151. deepfos/lib/decorator.py +310 -0
  152. deepfos/lib/deepchart.py +778 -0
  153. deepfos/lib/deepux.py +477 -0
  154. deepfos/lib/discovery.py +273 -0
  155. deepfos/lib/edb_lexer.py +789 -0
  156. deepfos/lib/eureka.py +156 -0
  157. deepfos/lib/filterparser.py +751 -0
  158. deepfos/lib/httpcli.py +106 -0
  159. deepfos/lib/jsonstreamer.py +80 -0
  160. deepfos/lib/msg.py +394 -0
  161. deepfos/lib/nacos.py +225 -0
  162. deepfos/lib/patch.py +92 -0
  163. deepfos/lib/redis.py +241 -0
  164. deepfos/lib/serutils.py +181 -0
  165. deepfos/lib/stopwatch.py +99 -0
  166. deepfos/lib/subtask.py +572 -0
  167. deepfos/lib/sysutils.py +703 -0
  168. deepfos/lib/utils.py +1003 -0
  169. deepfos/local.py +160 -0
  170. deepfos/options.py +670 -0
  171. deepfos/translation.py +237 -0
  172. deepfos-1.1.60.dist-info/METADATA +33 -0
  173. deepfos-1.1.60.dist-info/RECORD +175 -0
  174. deepfos-1.1.60.dist-info/WHEEL +5 -0
  175. deepfos-1.1.60.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1254 @@
1
+ import functools
2
+ from contextlib import contextmanager
3
+ from itertools import chain
4
+ from typing import List, Dict, Iterable, Any, Union, Type, Tuple, Optional, TYPE_CHECKING
5
+ from collections import Counter
6
+
7
+ import pandas as pd
8
+ from loguru import logger
9
+
10
+ from deepfos.element.rolestrategy import RoleStrategy, AsyncRoleStrategy
11
+ from pydantic import parse_obj_as, Field
12
+ from multidict import CIMultiDict
13
+
14
+ from .base import ElementBase, SyncMeta
15
+ from .datatable import AsyncDataTableMySQL, get_table_class, T_AsyncDatatableClass
16
+ from deepfos.algo.graph import DAGraph
17
+ from deepfos.lib.decorator import cached_property
18
+ from deepfos.api.dimension import DimensionAPI
19
+ from deepfos.api.models.dimension import (
20
+ DimensionMembersDto,
21
+ DimensionInfoSw,
22
+ DimensionMemberChangeSaveSw,
23
+ DimensionMemberOperationSw,
24
+ DimensionMemberSaveDto as DimMemberSave,
25
+ DimensionMemberListDto as DimInfo,
26
+ Dimension as DimensionModel,
27
+ DimensionMemberByNameFunctionDto as DimQueryExpr,
28
+ DimensionMemberBean,
29
+ )
30
+ from deepfos.api.V1_2.models.dimension import (
31
+ DimensionMemberByNameFunctionDto as DimQueryExpr1_2,
32
+ MultiEntityConfigResult,
33
+ DimensionMemberTreeSaveDto,
34
+ DimensionMemberOperationDto
35
+ )
36
+ from deepfos.exceptions import DimensionSaveError, MemberNotFoundError
37
+ from deepfos.lib.utils import unpack_expr, CIEnum, get_language_key_map
38
+ from deepfos.lib.asynchronous import future_property
39
+ from deepfos.lib.constant import (
40
+ ROOT, ACCEPT_LANS, DFLT_NAME_COLUMN,
41
+ DFLT_PNAME_COLUMN, SHAREDMEMBER, INDEX_FIELD,
42
+ SHAREDMEMBERV12, DFLT_PNAME_COLUMN_V12
43
+ )
44
+
45
+ __all__ = [
46
+ 'DimMember',
47
+ 'DimMemberV12',
48
+ 'Dimension',
49
+ 'AsyncDimension'
50
+ ]
51
+
52
+
53
+ class Strategy(CIEnum):
54
+ full_replace = 'full_replace'
55
+ incr_replace = 'incr_replace'
56
+ keep_old = 'keep_old'
57
+
58
+
59
+ class DimMember(DimensionMemberOperationSw):
60
+ is_active = True
61
+ is_modula = False
62
+ aggweight = 1.0
63
+ sharedmember = False
64
+ datatype = "NUMBER"
65
+ parent_name = ROOT
66
+
67
+
68
+ class DimMemberV12(DimensionMemberOperationDto):
69
+ isActive = True
70
+ aggweight = 1.0
71
+ sharedMember = False
72
+ dataType = "NUMBER"
73
+ parentName = ROOT
74
+
75
+ @property
76
+ def parent_name(self):
77
+ return self.parentName
78
+
79
+ @property
80
+ def sharedmember(self):
81
+ return self.sharedMember
82
+
83
+ @property
84
+ def origin_name(self):
85
+ return self.originName
86
+
87
+ @property
88
+ def origin_parent_name(self):
89
+ return self.originParentName
90
+
91
+ @property
92
+ def is_active(self):
93
+ return self.isActive
94
+
95
+ @property
96
+ def accounttype(self):
97
+ return self.accountType
98
+
99
+ @property
100
+ def datatype(self):
101
+ return self.dataType
102
+
103
+ @property
104
+ def end_period(self):
105
+ return self.endPeriod
106
+
107
+ @property
108
+ def end_year(self):
109
+ return self.endYear
110
+
111
+ @property
112
+ def flowtype(self):
113
+ return self.flowType
114
+
115
+ @property
116
+ def halfyear(self):
117
+ return self.halfYear
118
+
119
+ @property
120
+ def local_currency(self):
121
+ return self.localCurrency
122
+
123
+ @property
124
+ def period_level(self):
125
+ return self.periodLevel
126
+
127
+ @property
128
+ def start_year(self):
129
+ return self.startYear
130
+
131
+ @property
132
+ def start_period(self):
133
+ return self.startPeriod
134
+
135
+ @property
136
+ def sort_col(self):
137
+ return self.sortCol
138
+
139
+
140
+ ADD = 'add'
141
+ UPD = 'update'
142
+ DEL = 'delete'
143
+
144
+ OPE = 'operation'
145
+
146
+ V10_TO_V12 = {
147
+ 'accounttype': 'accountType',
148
+ 'datatype': 'dataType',
149
+ 'end_period': 'endPeriod',
150
+ 'end_year': 'endYear',
151
+ 'flowtype': 'flowType',
152
+ 'halfyear': 'halfYear',
153
+ 'is_active': 'isActive',
154
+ 'local_currency': 'localCurrency',
155
+ 'parent_name': 'parentName',
156
+ 'period_level': 'periodLevel',
157
+ 'origin_name': 'originName',
158
+ 'origin_parent_name': 'originParentName',
159
+ 'sharedmember': 'sharedMember',
160
+ 'start_period': 'startPeriod',
161
+ 'start_year': 'startYear',
162
+ 'sort_col': 'sortCol',
163
+ 'is_modula': 'is_modula',
164
+ 'is_base': 'is_base',
165
+ 'is_calculated': 'is_calculated',
166
+ 'formula': 'formula',
167
+ }
168
+
169
+
170
+ # -----------------------------------------------------------------------------
171
+ # core
172
+ class AsyncDimension(ElementBase[DimensionAPI]):
173
+ """维度
174
+
175
+ Args:
176
+ strict: 是否开启严格校验模式。如果开启,会查询系统中存在的所有维度,并且基于系统维度作完整性校验。会损失性能
177
+ """
178
+ api: DimensionAPI
179
+
180
+ def __init__(
181
+ self,
182
+ element_name: str,
183
+ folder_id: str = None,
184
+ path: str = None,
185
+ strict: bool = False,
186
+ server_name: str = None,
187
+ ):
188
+ self._add_memo = CIMultiDict()
189
+ self._del_memo = CIMultiDict()
190
+ self._upd_memo = CIMultiDict()
191
+ self.__member_memo = None if strict else CIMultiDict()
192
+ self._strict = strict
193
+ super().__init__(element_name, folder_id, path, server_name)
194
+
195
+ @cached_property
196
+ def _datatable_class(self) -> T_AsyncDatatableClass:
197
+ if (server_name := self.meta.databaseServerName) is None:
198
+ return AsyncDataTableMySQL
199
+
200
+ return get_table_class(server_name, sync=False)
201
+
202
+ @cached_property
203
+ def table_closure(self):
204
+ """维度层级关系表
205
+
206
+ Warnings:
207
+ 储存维度的层级关系,不要随意修改此表数据!
208
+
209
+ """
210
+ meta = self.meta
211
+ return self._datatable_class(
212
+ element_name=meta.tcServerName,
213
+ folder_id=meta.tcFolderId,
214
+ table_name=meta.table_closure,
215
+ server_name=meta.databaseServerName
216
+ )
217
+
218
+ @cached_property
219
+ def table_dimension(self):
220
+ """维度数据表
221
+
222
+ Warnings:
223
+ 储存维度的主数据,不要随意修改此表数据!
224
+ """
225
+ meta = self.meta
226
+ return self._datatable_class(
227
+ element_name=meta.tdServerName,
228
+ folder_id=meta.tdFolderId,
229
+ table_name=meta.table_dimension,
230
+ server_name=meta.databaseServerName
231
+ )
232
+
233
+ @property
234
+ def _member_memo(self) -> CIMultiDict[DimensionMemberOperationSw]:
235
+ if self.__member_memo is None:
236
+ self.__member_memo = memo = CIMultiDict()
237
+
238
+ for mbr in self.members:
239
+ memo.add(mbr.name, mbr)
240
+
241
+ return self.__member_memo
242
+
243
+ @cached_property
244
+ def is_v1_0_api(self) -> bool:
245
+ return self.api.version == (1, 0)
246
+
247
+ @future_property(on_demand=True)
248
+ async def members(self) -> List[DimensionMembersDto]:
249
+ """当前维度的所有成员
250
+
251
+ Note:
252
+ 只会返回系统中存在的维度成员。通过 :meth:`add_member`
253
+ 或 :meth:`delete_member` 本地修改维度成员并不会会影响此属性的值。
254
+ """
255
+ api = await self.wait_for('async_api')
256
+ element_info = await self.wait_for('element_info')
257
+ dim_info = DimInfo.construct_from(
258
+ element_info,
259
+ name=self.element_name
260
+ )
261
+ if api.version == api.default_version:
262
+ mbr = await api.custom.select_dimension_member_list(dim_info)
263
+ else:
264
+ mbr = await api.extra.dimension_custom_select_dimension_member_list(dim_info)
265
+ return parse_obj_as(List[DimensionMembersDto], mbr['memberList'])
266
+
267
+ @future_property
268
+ async def meta(self) -> DimensionModel:
269
+ """
270
+ 当前维度的元信息
271
+ """
272
+ api = await self.wait_for('async_api')
273
+ ele_info = await self.wait_for('element_info')
274
+ if api.version == api.default_version:
275
+ return await api.query.open_dimension_info_by_id(ele_info)
276
+ return await api.extra.dimension_query_open_dimension_info_by_id(ele_info)
277
+
278
+ def add_member(
279
+ self,
280
+ *members: Union[DimMember, DimMemberV12],
281
+ silent: bool = False
282
+ ) -> 'AsyncDimension':
283
+ """增加维度成员,
284
+
285
+ 新增的维度成员暂时维护在当前的维度对象中,
286
+ 调用 :meth:`save` 之后生效。
287
+
288
+ Args:
289
+ *members: 要增加的维度成员
290
+ silent: 当维度成员名重复时,是报错还是静默处理。
291
+ 默认False,即报错处理。
292
+
293
+ Returns:
294
+ self
295
+ """
296
+ name_conflict = []
297
+ for mbr in members:
298
+ name = mbr.name
299
+ if name in self._member_memo or name in self._add_memo:
300
+ if not mbr.sharedmember:
301
+ name_conflict.append(name)
302
+ else:
303
+ mbr.operation = ADD
304
+ self._add_memo.add(name, mbr)
305
+ else:
306
+ mbr.operation = ADD
307
+ self._add_memo.add(name, mbr)
308
+
309
+ if not silent and name_conflict:
310
+ raise NameError(f"Member: {name_conflict} already exists.")
311
+ return self
312
+
313
+ def delete_member(self, *members: str, silent: bool = True) -> 'AsyncDimension':
314
+ """删除维度成员
315
+
316
+ 如果删除的成员是共享节点,将删除所有共享节点。
317
+ 调用 :meth:`save` 之后生效。
318
+
319
+ Args:
320
+ *members: 要删除的维度 **成员名**
321
+ silent: 当维度成员不存在时,是报错还是静默处理。
322
+ 默认True, 即静默处理。
323
+
324
+ Returns:
325
+ self
326
+
327
+ See Also:
328
+ :meth:`delete_shared_memeber`
329
+
330
+ """
331
+ mbr_not_found = []
332
+
333
+ for member in members:
334
+ self._del_memo.add(member, DimMember(name=member, operation=DEL))
335
+
336
+ if member in self._member_memo:
337
+ del self._member_memo[member]
338
+ elif member in self._upd_memo:
339
+ del self._upd_memo[member]
340
+ elif member in self._add_memo:
341
+ del self._add_memo[member]
342
+ else:
343
+ mbr_not_found.append(member)
344
+
345
+ if self._strict and (not silent) and mbr_not_found:
346
+ raise MemberNotFoundError(f"Member {mbr_not_found} does not exist.")
347
+ return self
348
+
349
+ def delete_shared_memeber(
350
+ self,
351
+ member: str,
352
+ parent_name: Union[str, List[str]] = ROOT,
353
+ silent: bool = True
354
+ ) -> 'AsyncDimension':
355
+ """删除共享维度成员
356
+
357
+ 删除指定父节点下的共享成员节点,
358
+ 调用 :meth:`save` 之后生效。
359
+
360
+ Args:
361
+ member: 要删除的维度 **成员名**
362
+ parent_name: 共享节点的父节点名,默认为#root
363
+ silent: 当维度成员不存在时,是报错还是静默处理。
364
+ 默认True, 即静默处理。
365
+
366
+ """
367
+ target_found = False
368
+ new_value = []
369
+ if isinstance(parent_name, str):
370
+ parent_name = [parent_name]
371
+
372
+ for dim_mbr in self._member_memo.getall(member, []):
373
+ if dim_mbr.parent_name in parent_name:
374
+ target_found = True
375
+ else:
376
+ new_value.append((member, dim_mbr))
377
+
378
+ if not target_found and not silent:
379
+ raise MemberNotFoundError(
380
+ f"Cannot find shared member {member} with parent name: {parent_name}.")
381
+
382
+ del self._member_memo[member]
383
+ self._member_memo.extend(new_value)
384
+ return self
385
+
386
+ def reorder_members(
387
+ self,
388
+ memo: CIMultiDict[Union[DimMember, DimMemberV12]],
389
+ complete: bool
390
+ ) -> CIMultiDict[Union[DimMember, DimMemberV12]]:
391
+ """成员重排序
392
+
393
+ 对维度成员按照先父节点后子节点的顺序排序
394
+
395
+ Args:
396
+ memo: 需要重排序的维度成员
397
+ complete: memo里的成员是否是全部维度成员
398
+ """
399
+ parents = set(mbr.parent_name for mbr in memo.values())
400
+ parents.discard(ROOT)
401
+ all_mbr = set(memo.keys())
402
+ unknown = parents - all_mbr
403
+
404
+ if unknown and complete:
405
+ raise DimensionSaveError(f"Missing parent node: {unknown}.")
406
+
407
+ if self._strict:
408
+ mbr_memo = self._member_memo
409
+ if any((node := item) not in mbr_memo for item in unknown):
410
+ raise DimensionSaveError(f"Missing parent node: {node}.")
411
+
412
+ to_reorder = {}
413
+ for member in memo.values():
414
+ if member.parent_name == ROOT:
415
+ to_reorder[member.name] = []
416
+ else:
417
+ to_reorder.setdefault(member.name, []).append(member.parent_name)
418
+
419
+ ordered_keys = reversed(DAGraph(list(to_reorder.items())).topsort())
420
+
421
+ ordered_mbrs = CIMultiDict()
422
+
423
+ for k in ordered_keys:
424
+ if k not in memo:
425
+ continue
426
+
427
+ for v in memo.getall(k):
428
+ ordered_mbrs.add(k, v)
429
+
430
+ # 补充孤立成员
431
+ for k in set(memo.keys()).difference(set(ordered_mbrs.keys())):
432
+ for v in memo.getall(k):
433
+ ordered_mbrs.add(k, v)
434
+
435
+ return ordered_mbrs
436
+
437
+ async def save(
438
+ self,
439
+ reorder: bool = False
440
+ ):
441
+ """保存维度
442
+
443
+ 将对当前维度的修改保存至系统。
444
+
445
+ Args:
446
+ reorder: 是否对保存的元素进行重排序,将会损失性能。
447
+
448
+ Note:
449
+ 由于目前维度保存接口的缺陷,如果保存数据中,父节点出现在
450
+ 子节点之后,父节点将不会被正常识别,从而导致保存失败。
451
+ 此方法最终调用保存接口的维度顺序将会和 :meth:`add_member` 的顺序一致,
452
+ 因此如果使用者自身能够保证维度的出现顺序,那么可以不使用reorder,
453
+ 否则应当开启。
454
+
455
+ """
456
+ if reorder:
457
+ self._add_memo = self.reorder_members(self._add_memo, complete=False)
458
+
459
+ await self._finalize_update()
460
+
461
+ dim_members = []
462
+ for v in chain(
463
+ self._add_memo.values(),
464
+ self._upd_memo.values(),
465
+ self._del_memo.values()
466
+ ):
467
+ mbr = v
468
+ if self.is_v1_0_api:
469
+ mbr.index = mbr.index or 0
470
+ else:
471
+ mbr = self._maybe_rename_fields(v)
472
+ dim_members.append(mbr.dict(exclude_none=True))
473
+
474
+ if len(dim_members) > 0:
475
+ if self.is_v1_0_api:
476
+ payload = DimensionMemberChangeSaveSw.construct_from(
477
+ dimensionInfo=DimensionInfoSw(
478
+ name=self.element_info.elementName,
479
+ folderId=self.element_info.folderId,
480
+ isOnlyCheck='0'
481
+ ),
482
+ dimensionMemberList=dim_members
483
+ )
484
+ r = await self._incr_save_members(payload)
485
+ else:
486
+ r = await self._save_tree(
487
+ parse_obj_as(List[DimensionMemberOperationDto], dim_members)
488
+ )
489
+ self._add_memo.clear()
490
+ self._del_memo.clear()
491
+ self._upd_memo.clear()
492
+ return r
493
+
494
+ def _maybe_rename_fields(
495
+ self,
496
+ mbr: Union[
497
+ DimensionMemberOperationSw,
498
+ DimensionMemberOperationDto,
499
+ Dict[str, str]
500
+ ]
501
+ ) -> Union[
502
+ DimensionMemberOperationSw,
503
+ DimensionMemberOperationDto,
504
+ Dict[str, str]
505
+ ]:
506
+ """
507
+ Maybe rename fields in V1.0 dimension mbr model
508
+ to v1.1 & v1.2 model when current api is v1.1 or v1.2
509
+ """
510
+ if self.is_v1_0_api:
511
+ return mbr
512
+
513
+ if isinstance(mbr, DimensionMemberOperationDto):
514
+ return mbr
515
+
516
+ return_model = False
517
+ if isinstance(mbr, DimensionMemberOperationSw):
518
+ return_model = True
519
+ mbr = mbr.dict(exclude_none=True)
520
+
521
+ for old, new in V10_TO_V12.items():
522
+ if old in mbr:
523
+ mbr[new] = mbr.pop(old)
524
+
525
+ if return_model:
526
+ return DimMemberV12.construct_from(**mbr)
527
+
528
+ return mbr
529
+
530
+ async def _incr_save_members(self, payload: DimensionMemberChangeSaveSw):
531
+ resp = await self.async_api.save.incremental(payload)
532
+ if resp.errors:
533
+ raise DimensionSaveError(f"Failed to save dimension. Detail: {resp}")
534
+ return resp
535
+
536
+ async def query(
537
+ self,
538
+ expression: str,
539
+ fields: Union[List[str], Tuple[str]] = None,
540
+ as_model: bool = True,
541
+ role: Union[RoleStrategy, str] = None,
542
+ role_name: str = None,
543
+ role_group: str = None,
544
+ multi_entity_config: List[Dict] = None
545
+ ) -> Union[
546
+ List[DimensionMemberBean], List[Dict[str, Any]], List[MultiEntityConfigResult]
547
+ ]:
548
+ """查询成员
549
+
550
+ 根据维度表达式查询维度成员
551
+
552
+ Args:
553
+ expression: 维度表达式
554
+ fields: 需要查询的字段
555
+ as_model: 是否把返回的结果转化为pydantic model
556
+ role: 权限方案元素对象或权限方案名,提供时将带权限方案信息查询
557
+ role_name: 角色名,默认为当前用户的角色信息
558
+ role_group: 角色组名,默认为当前用户的角色信息
559
+ multi_entity_config: 实体维度配置表过滤条件
560
+
561
+ Important:
562
+ - ``expression`` 可以不包含维度名,即: 如果当前维度是Entity,
563
+ ``Entity{Base(#root,0)}`` 和 ``Base(#root,0)`` 是等价的。
564
+ - 为保证接口正常调用, ``fields`` 将自动包含 ``name`` 属性。
565
+ - ``as_model`` 参数可以影响本方法的返回类型,可以根据使用需要指定。
566
+
567
+ .. admonition:: 示例
568
+
569
+ 如果希望将查询出的成员作为对象操作,推荐使用默认参数查询:
570
+
571
+ .. code-block:: python
572
+
573
+ dim = Dimension('dim_example')
574
+ mbrs = dim.query('Base(#root,0)', fields=['ud1'])
575
+ mbr = mbrs[0]
576
+ ud1 = mbr.ud1
577
+ name = mbr.name
578
+ ...
579
+
580
+ **注意:未查询的属性也能访问,但值固定是None**
581
+
582
+ .. code-block:: python
583
+ :emphasize-lines: 1
584
+
585
+ assert mbrs.ud2 is None
586
+
587
+ 可以把上述查询结果转换为dataframe:
588
+
589
+ .. code-block:: python
590
+ :emphasize-lines: 1
591
+
592
+ data = (mbr.dict(exclude_unset=True) for mbr in mbrs)
593
+ df = pd.DataFrame(data=data)
594
+
595
+ 注意需要参数 ``exclude_unset=True`` 排除未查询到的数据。
596
+
597
+ 如果希望把查询数据转换为dataframe,更推荐使用以下方式:
598
+
599
+ .. code-block:: python
600
+ :emphasize-lines: 1,2
601
+
602
+ dim = Dimension('dim_example')
603
+ mbrs = dim.query('Base(#root,0)', fields=['ud1'], as_model=False)
604
+ # 注意此时返回的数据类型不同
605
+ assert isinstance(mbrs, List[dict])
606
+ df = pd.DataFrame(data=mbrs)
607
+
608
+ 如果希望带权限方案查询,可用 ``role`` 、 ``role_name`` 以及 ``role_group`` 参数表示权限方案信息
609
+
610
+ .. code-block:: python
611
+
612
+ dim = Dimension('component')
613
+ mbrs = dim.query('Descendant(East,0)', role='test_role', role_name='nonsense')
614
+
615
+ 此时会以权限方案test_role中的nonsense角色作为查询的角色信息,查询维度表达式:component{Descendant(East,0)}
616
+
617
+ **注意:role_name和role_group若提供,则提供其一即可**
618
+
619
+ Returns:
620
+ - 如果 ``as_model == True``, 返回 :obj:`List[DimensionMemberBean]` 类型,
621
+ 不在查询fields中的属性将会被设置为None。
622
+ - 如果 ``as_model == False``,返回 :obj:`List[Dict[str, Any]]` 类型,
623
+ 内部的字典将不包含未查询的属性
624
+
625
+ """
626
+ payload = self._get_query_payload(
627
+ expression, fields, role, role_name, role_group, multi_entity_config
628
+ )
629
+ if self.is_v1_0_api:
630
+ api_call = self.async_api.query.select_dimension_member_by_name_function
631
+ elif (
632
+ self.api.version == (1, 2)
633
+ and as_model and multi_entity_config is not None
634
+ ):
635
+ api_call = functools.partial(
636
+ self.async_api.extra.dimension_query_select_dimension_member_by_name_function,
637
+ resp_model=List[MultiEntityConfigResult]
638
+ )
639
+ else:
640
+ api_call = self.async_api.extra.dimension_query_select_dimension_member_by_name_function
641
+ if as_model:
642
+ return await api_call(payload)
643
+ else:
644
+ return await api_call(payload, resp_model=List[dict])
645
+
646
+ def _get_query_payload(
647
+ self,
648
+ expression,
649
+ fields,
650
+ role=None,
651
+ role_name=None,
652
+ role_group=None,
653
+ multi_entity_config=None
654
+ ):
655
+ if self.api.version == (1, 2):
656
+ payload_cls = DimQueryExpr1_2
657
+ else:
658
+ payload_cls = DimQueryExpr
659
+
660
+ role_params = {}
661
+
662
+ if role is not None:
663
+ if isinstance(role, str):
664
+ role = AsyncRoleStrategy(element_name=role)
665
+
666
+ rs_mapping = 1
667
+ for dim in role.meta.dimensions:
668
+ if (
669
+ dim.elementName == self.element_name
670
+ and dim.folderId == self.element_info.folderId
671
+ ):
672
+ break
673
+ rs_mapping += 1
674
+ if rs_mapping > len(role.meta.dimensions):
675
+ raise ValueError(
676
+ f"Current Dimension: {self.element_name} doesn't exist in "
677
+ f"role strategy: {role.element_name}."
678
+ )
679
+
680
+ role_params.update({'roleFolderId': role.element_info.folderId,
681
+ 'rsMapping': rs_mapping,
682
+ 'rsName': role.element_name})
683
+
684
+ if role_name or role_group:
685
+ role_params.update({'role': role_name, 'rolegroup': role_group})
686
+
687
+ multi_entity_config_list = {}
688
+ if multi_entity_config is not None and self.api.version == (1, 2):
689
+ multi_entity_config_list['multiEntityConfigSearchDTOList'] = multi_entity_config
690
+
691
+ if fields is not None:
692
+ fields = set(fields).union([DFLT_NAME_COLUMN])
693
+ else:
694
+ fields = []
695
+ dim, body = unpack_expr(expression, silent=True)
696
+ if dim is None:
697
+ dim = self.element_name
698
+ expr = "%s{%s}" % (dim, body)
699
+ payload = payload_cls.construct_from(
700
+ dimensionMemberNames=expr,
701
+ duplicate=True,
702
+ ignoreIllegalMember=True,
703
+ resultString=','.join(fields),
704
+ web=False,
705
+ folderId=self.element_info.folderId,
706
+ reverse_order='0',
707
+ **role_params,
708
+ **multi_entity_config_list
709
+ )
710
+ return payload
711
+
712
+ def update(self, member_name: str, **attrs: Any):
713
+ """更新维度成员
714
+
715
+ 调用 :meth:`save` 之后生效。
716
+
717
+ Args:
718
+ member_name: 维度成员名
719
+
720
+ **attrs: 成员属性,可接受参数参考:
721
+
722
+ - 维度1.0 : :class:`DimMember`
723
+ - 维度1.1或1.2 : :class:`DimMemberV12`
724
+
725
+ Returns:
726
+ self
727
+ """
728
+ attrs = self._maybe_rename_fields(attrs)
729
+ if member_name in self._add_memo:
730
+ upd_mbr = self._add_memo[member_name]
731
+ if isinstance(upd_mbr, DimensionMemberOperationSw):
732
+ self._add_memo[member_name] = DimMember.construct_from(
733
+ upd_mbr, **attrs
734
+ )
735
+ else:
736
+ self._add_memo[member_name] = DimMemberV12.construct_from(
737
+ upd_mbr, **attrs
738
+ )
739
+ elif member_name in self._del_memo:
740
+ raise MemberNotFoundError(f"Member: {member_name} does not exist.")
741
+ elif self._strict and member_name not in self._member_memo:
742
+ raise MemberNotFoundError(f"Member: {member_name} does not exist.")
743
+ else:
744
+ self._upd_memo.add(member_name, attrs)
745
+
746
+ return self
747
+
748
+ async def load_dataframe(
749
+ self,
750
+ dataframe: pd.DataFrame,
751
+ strategy: Union[Strategy, str] = Strategy.incr_replace,
752
+ reorder: bool = False,
753
+ **langugage_keys: str
754
+ ):
755
+ """保存 ``DataFrame`` 数据至维度
756
+
757
+ 此方法不同于 :meth:`add_member`,:meth:`delete_member`
758
+ 等方法,保存结果将直接反映至系统,不需要再调用save。
759
+
760
+ Args:
761
+ dataframe: 包含维度数据的DataFrame
762
+ strategy: 数据保存策略
763
+ reorder: 是否对保存的元素进行重排序,将会损失性能。
764
+ **langugage_keys: 维度成员描述(多语言)对应的列名
765
+
766
+ Note:
767
+ 1. 数据保存策略可选参数如下:
768
+
769
+ +--------------+--------------------------------------------+
770
+ | 参数 | 说明 |
771
+ +==============+============================================+
772
+ | full_replace | 完全替换所有维度成员。 |
773
+ | | 此策略将会删除所有已有维度成员, |
774
+ | | 以dataframe为数据源新建维度成员。 |
775
+ +--------------+--------------------------------------------+
776
+ | incr_replace | 增量替换维度成员。 |
777
+ | | 此策略不会删除已有维度成员。 |
778
+ | | 在保存过程中,如果遇到成员名重复的情况, |
779
+ | | 会以dataframe数据为准,覆盖已有成员。 |
780
+ +--------------+--------------------------------------------+
781
+ | keep_old | 保留已有维度成员。 |
782
+ | | 此策略在保存过程中,遇到成员名重复的情况, |
783
+ | | 会保留已有成员。其他与incr_replace相同。 |
784
+ +--------------+--------------------------------------------+
785
+
786
+ 2. 目前描述支持两种语言: ``zh-cn, en``,此方法默认会在dataframe中寻找
787
+ 名为 ``'language_zh-cn', 'language_en'`` 的列,将其数据作为对应
788
+ 语言的描述。如果想改变这种默认行为,比如希望用'name'列作为中文语言描述,
789
+ 可以传入关键字参数: ``language_zh_cn='name'``。
790
+
791
+ Warnings:
792
+ - 目前由于技术原因,如果要保存共享维度成员,必须使用full_replace策略。
793
+ - 如果传入的dataframe不含index列,将自动以0填充该列。
794
+
795
+ """
796
+ if dataframe.empty:
797
+ return
798
+
799
+ strategy = Strategy[strategy]
800
+ df = dataframe.copy().reset_index(drop=True)
801
+ if INDEX_FIELD not in df.columns and self.is_v1_0_api:
802
+ df[INDEX_FIELD] = 0
803
+
804
+ # validate dataframe
805
+ _validate_df_for_dimension(df)
806
+
807
+ # create language columns
808
+ language_map = get_language_key_map(langugage_keys)
809
+ for lan, key in language_map.items():
810
+ if key in df.columns:
811
+ df[lan] = df[key]
812
+
813
+ # 合并描述列
814
+ lan_columns = list(set(df.columns).intersection(ACCEPT_LANS))
815
+ if lan_columns:
816
+ df['multilingual'] = df[lan_columns].to_dict(orient='records')
817
+
818
+ mbrcls = DimMember if self.is_v1_0_api else DimMemberV12
819
+
820
+ if not self.is_v1_0_api:
821
+ df = df.rename(columns=V10_TO_V12)
822
+
823
+ # pick up valid columns
824
+ valid_columns = list(
825
+ set(mbrcls.__fields__).intersection(df.columns)
826
+ )
827
+
828
+ # save replace
829
+ if strategy is Strategy.full_replace:
830
+ member_list = df[valid_columns].to_dict(orient='records')
831
+ return await self._save_replace(reorder, member_list)
832
+
833
+ # split dataframe by existed data
834
+ upd_df, add_df = await self._split_dimension_df(df[valid_columns])
835
+
836
+ # modify operation column
837
+ with self._local_memo():
838
+ if strategy is Strategy.incr_replace:
839
+ for record in upd_df.to_dict(orient='records'):
840
+ self._upd_memo.add(record[DFLT_NAME_COLUMN], record)
841
+
842
+ for record in add_df.to_dict(orient='records'):
843
+ self._add_memo.add(
844
+ record[DFLT_NAME_COLUMN],
845
+ mbrcls.construct_from(**record, operation=ADD)
846
+ )
847
+
848
+ return await self.save(reorder=reorder)
849
+
850
+ async def _save_replace(
851
+ self, reorder: bool, member_list: List[Dict[str, str]]
852
+ ):
853
+ mbrcls = DimMember if self.is_v1_0_api else DimMemberV12
854
+
855
+ if reorder:
856
+ mbrs = CIMultiDict()
857
+ for rec in member_list:
858
+ mbrs.add(rec[DFLT_NAME_COLUMN], mbrcls.construct_from(**rec))
859
+ mbrs = self.reorder_members(mbrs, complete=True)
860
+ member_list = [mbr.dict(exclude_none=True) for mbr in mbrs.values()]
861
+
862
+ payload = DimMemberSave.construct_from(
863
+ self.element_info,
864
+ dimensionName=self.element_name,
865
+ increment=0,
866
+ dimensionMemberList=member_list
867
+ )
868
+ if self.is_v1_0_api:
869
+ resp = await self.async_api.member.save(payload)
870
+ elif self.api.version == (1, 1):
871
+ resp = await self.async_api.save.save(payload)
872
+ else:
873
+ resp = await self.async_api.member.refactor_dimension_member_save(payload)
874
+
875
+ if not resp.success:
876
+ raise DimensionSaveError(f"Failed to save dimension. Detail: {resp}")
877
+
878
+ return resp
879
+
880
+ async def _finalize_update(self):
881
+ upd_memo = self._upd_memo
882
+ if not upd_memo:
883
+ mbrs = []
884
+ else:
885
+ mbrs = await self.query(";".join(upd_memo.keys()))
886
+ new_memo = CIMultiDict()
887
+
888
+ for mbr in mbrs:
889
+ # Use DimMember since query response model fits
890
+ upd_mbr = DimMember.construct_from(mbr)
891
+ upd_mbr.operation = UPD
892
+ upd_mbr.origin_name = upd_mbr.name
893
+ upd_mbr.origin_parent_name = upd_mbr.parent_name
894
+
895
+ upd_mbr = self._maybe_rename_fields(upd_mbr)
896
+
897
+ for attrs in upd_memo.getall(upd_mbr.name, []):
898
+ if DFLT_NAME_COLUMN in attrs:
899
+ # 移除被改名的原始成员
900
+ self._member_memo.pop(upd_mbr.origin_name, None)
901
+
902
+ # 更新属性
903
+ for attr, val in attrs.items():
904
+ origin = getattr(upd_mbr, attr, None)
905
+ if isinstance(origin, dict):
906
+ if not isinstance(val, dict):
907
+ raise TypeError(
908
+ f"Update {attr} failed. "
909
+ f"Expect dict type, got {type(val)}.")
910
+ origin.update(val)
911
+ else:
912
+ setattr(upd_mbr, attr, val)
913
+
914
+ new_memo.add(upd_mbr.name, upd_mbr)
915
+
916
+ self._upd_memo = new_memo
917
+
918
+ @contextmanager
919
+ def _local_memo(self):
920
+ upd_bak = self._upd_memo
921
+ add_bak = self._add_memo
922
+ del_bak = self._del_memo
923
+ self._upd_memo = CIMultiDict()
924
+ self._add_memo = CIMultiDict()
925
+ self._del_memo = CIMultiDict()
926
+ try:
927
+ yield
928
+ finally:
929
+ self._upd_memo = upd_bak
930
+ self._add_memo = add_bak
931
+ self._del_memo = del_bak
932
+
933
+ async def _split_dimension_df(
934
+ self,
935
+ df: pd.DataFrame
936
+ ) -> Tuple[pd.DataFrame, pd.DataFrame]:
937
+ shared_mbr_col = SHAREDMEMBER
938
+ pname_col = DFLT_PNAME_COLUMN
939
+ if not self.is_v1_0_api:
940
+ shared_mbr_col = SHAREDMEMBERV12
941
+ pname_col = DFLT_PNAME_COLUMN_V12
942
+
943
+ if shared_mbr_col not in df.columns:
944
+ if self._strict:
945
+ existed_mbrs = self._member_memo
946
+ else:
947
+ name_col = getattr(self.table_dimension.table, DFLT_NAME_COLUMN)
948
+ dim_names = await self.table_dimension.select(
949
+ [DFLT_NAME_COLUMN],
950
+ distinct=True,
951
+ where=name_col.isin(df[DFLT_NAME_COLUMN].tolist())
952
+ )
953
+ existed_mbrs = set(dim_names[DFLT_NAME_COLUMN])
954
+ existed_idx = df[DFLT_NAME_COLUMN].isin(existed_mbrs)
955
+ return df.loc[existed_idx], df.loc[~existed_idx]
956
+
957
+ # -----------------------------------------------------------------------------
958
+ # has shared member
959
+ df_shared: pd.DataFrame = df[df[shared_mbr_col]]
960
+ if self._strict:
961
+ existed_mbrs = self._member_memo
962
+
963
+ dup_sharedmbrs = []
964
+ for idx, row in df_shared.iterrows():
965
+ for mbr in self._member_memo.getall(row[DFLT_NAME_COLUMN], []):
966
+ if mbr.sharedmember and mbr.parent_name == row[pname_col]:
967
+ dup_sharedmbrs.append(idx)
968
+ break
969
+ else:
970
+ name_col = getattr(self.table_dimension.table, DFLT_NAME_COLUMN)
971
+ existed_df = await self.table_dimension.select(
972
+ [DFLT_NAME_COLUMN, DFLT_PNAME_COLUMN, SHAREDMEMBER],
973
+ distinct=True,
974
+ where=name_col.isin(df[DFLT_NAME_COLUMN].tolist())
975
+ )
976
+ existed_mbrs = set(existed_df[DFLT_NAME_COLUMN].tolist())
977
+
978
+ existed_shared: pd.DataFrame = existed_df.loc[
979
+ existed_df[SHAREDMEMBER],
980
+ [DFLT_NAME_COLUMN, DFLT_PNAME_COLUMN]
981
+ ]
982
+
983
+ shared_memo = set(tuple(item) for item in existed_shared.to_numpy())
984
+ dup_sharedmbrs = []
985
+ for idx, row in df_shared.iterrows():
986
+ if (row[DFLT_NAME_COLUMN], row[pname_col]) in shared_memo:
987
+ dup_sharedmbrs.append(idx)
988
+
989
+ # 在df中丢弃系统中已存在的共享节点
990
+ df_shared_remain = df_shared[~df_shared.index.isin(dup_sharedmbrs)]
991
+
992
+ df_not_shared = df[~df[shared_mbr_col]]
993
+ existed_idx = df_not_shared[DFLT_NAME_COLUMN].isin(existed_mbrs)
994
+ return df_not_shared.loc[existed_idx], \
995
+ pd.concat([df_not_shared.loc[~existed_idx], df_shared_remain])
996
+
997
+ async def to_deepmodel_object(
998
+ self,
999
+ object_name: str,
1000
+ expression: str = None,
1001
+ field_map: Dict[str, str] = None,
1002
+ full_replace: bool = False
1003
+ ):
1004
+ """维度成员导入至已有DeepModel对象
1005
+
1006
+ Args:
1007
+ object_name: DeepModel对象名
1008
+ expression: 可选,维度成员表达式,默认为维度成员全集
1009
+ field_map: 可选,维度成员属性名与DeepModel对象字段名的映射关系
1010
+ full_replace: 是否全量替换,是,则会先清空DeepModel对象内数据后再导入
1011
+
1012
+ """
1013
+ if expression is None:
1014
+ expression = "Descendant(#root,0)"
1015
+
1016
+ data = await self.query(expression, as_model=False)
1017
+ data = pd.DataFrame(data)
1018
+ if field_map:
1019
+ data = data.rename(columns=field_map)
1020
+
1021
+ # 父子关系与aggweight为层级关系所需信息
1022
+ parent = data[[DFLT_NAME_COLUMN, DFLT_PNAME_COLUMN, 'aggweight']]
1023
+ # parent_name为#root,即只作parent的记录
1024
+ parent = parent[~(parent[DFLT_PNAME_COLUMN] == '#root')]
1025
+ # 共享成员的父子关系重复,去重
1026
+ # 保证source -> target关系唯一性
1027
+ parent = parent.drop_duplicates(
1028
+ subset=[DFLT_NAME_COLUMN, DFLT_PNAME_COLUMN],
1029
+ keep='first'
1030
+ )
1031
+ parent = parent.rename(
1032
+ columns={DFLT_NAME_COLUMN: 'source', DFLT_PNAME_COLUMN: 'target'}
1033
+ )
1034
+ # 主数据排除共享成员
1035
+ data = data.drop_duplicates(subset=[DFLT_NAME_COLUMN], keep='first')
1036
+ from deepfos.element.deepmodel import AsyncDeepModel
1037
+ dm = AsyncDeepModel()
1038
+ async with dm.start_transaction():
1039
+ if full_replace:
1040
+ await dm.execute(f"delete {object_name}")
1041
+ await dm.insert_df(object_name, data, {'parent': parent})
1042
+
1043
+ async def sync_data(self):
1044
+ """等效于调用DeepModel创建的维度的数据同步"""
1045
+ if self.async_api.version == (1, 2):
1046
+ return await self.async_api.object.sync_data(
1047
+ folderId=self.element_info.folderId,
1048
+ name=self.element_name
1049
+ )
1050
+
1051
+ logger.warning("Only supported when Element Version is 1.2")
1052
+
1053
+ async def update_parent(
1054
+ self,
1055
+ member_name: str,
1056
+ origin_parent: str,
1057
+ new_parent: str,
1058
+ shared_member: bool = False
1059
+ ):
1060
+ """修改系统已有维度成员的父级成员
1061
+
1062
+ 此方法不同于 :meth:`add_member`,:meth:`delete_member`
1063
+ 等方法,保存结果将直接反映至系统,不需要再调用save。
1064
+
1065
+ Args:
1066
+ member_name: 成员名
1067
+ origin_parent: 原父级成员名
1068
+ new_parent: 新父级成员名
1069
+ shared_member: 是否为共享成员,默认否
1070
+
1071
+
1072
+ """
1073
+ mbrs = [
1074
+ DimensionMemberOperationDto.construct_from(
1075
+ originParentName=origin_parent,
1076
+ originName=member_name,
1077
+ name=member_name,
1078
+ parentName=new_parent,
1079
+ operation=UPD,
1080
+ sharedMember=shared_member
1081
+ )
1082
+ ]
1083
+ return await self._save_tree(mbrs)
1084
+
1085
+ async def update_parent_batch(
1086
+ self,
1087
+ dataframe: pd.DataFrame
1088
+ ):
1089
+ """批量修改系统已有维度成员的父级成员
1090
+
1091
+ 此方法不同于 :meth:`add_member`,:meth:`delete_member`
1092
+ 等方法,保存结果将直接反映至系统,不需要再调用save。
1093
+
1094
+ Args:
1095
+ dataframe: 批量修改信息的dataframe
1096
+
1097
+ Note:
1098
+ dataframe需包含的信息:
1099
+
1100
+ - member_name: 成员名
1101
+ - origin_parent: 原父级成员名
1102
+ - new_parent: 新父级成员名
1103
+
1104
+ 如需指定共享成员,则提供shared_member列用于区分是否为共享成员
1105
+
1106
+
1107
+ """
1108
+ if dataframe.empty:
1109
+ return
1110
+
1111
+ required_cols = {'member_name', 'origin_parent', 'new_parent'}
1112
+ if lacked := required_cols.difference(dataframe.columns):
1113
+ raise DimensionSaveError(
1114
+ f"Required cols for update parent: {lacked} lacked."
1115
+ )
1116
+
1117
+ fields = required_cols.union({'shared_member'})
1118
+ if 'shared_member' not in dataframe.columns:
1119
+ dataframe = dataframe.assign(shared_member=False)
1120
+
1121
+ records = dataframe[list(fields)].to_dict(orient="records")
1122
+ mbrs = []
1123
+ for record in records:
1124
+ mbrs.append(
1125
+ DimensionMemberOperationDto.construct_from(
1126
+ originParentName=record['origin_parent'],
1127
+ originName=record['member_name'],
1128
+ name=record['member_name'],
1129
+ parentName=record['new_parent'],
1130
+ operation=UPD,
1131
+ sharedMember=record['shared_member']
1132
+ )
1133
+ )
1134
+ return await self._save_tree(mbrs)
1135
+
1136
+ async def _save_tree(self, mbrs: List[DimensionMemberOperationDto]):
1137
+ if self.async_api.version == (1, 1):
1138
+ call_api = self.async_api.save.tree_save
1139
+ elif self.async_api.version == (1, 2):
1140
+ call_api = self.async_api.member.refactor_dimension_member_tree_save
1141
+ else:
1142
+ logger.warning("Only supported when Element Version is 1.1 or 1.2")
1143
+ return
1144
+
1145
+ payload = DimensionMemberTreeSaveDto.construct_from(
1146
+ self.element_info,
1147
+ dimensionName=self.element_name,
1148
+ dimensionMemberList=mbrs
1149
+ )
1150
+ resp = await call_api(payload)
1151
+ if resp.errors:
1152
+ raise DimensionSaveError(f"Failed to save dimension. Detail: {resp}")
1153
+
1154
+
1155
+ def _validate_df_for_dimension(df: pd.DataFrame):
1156
+ if DFLT_NAME_COLUMN not in df.columns:
1157
+ raise ValueError(f"Missing column [{DFLT_NAME_COLUMN}] in dataframe.")
1158
+
1159
+ if df[DFLT_NAME_COLUMN].hasnans:
1160
+ null_index = df.loc[df[DFLT_NAME_COLUMN].isna()].index.values
1161
+ raise ValueError(
1162
+ f"You have null value in dataframe. "
1163
+ f"column: [{DFLT_NAME_COLUMN}], index: {null_index}.")
1164
+
1165
+ if DFLT_PNAME_COLUMN not in df.columns:
1166
+ raise ValueError(f"Missing column [{DFLT_PNAME_COLUMN}] in dataframe.")
1167
+
1168
+ if SHAREDMEMBER in df.columns:
1169
+ unique_df = df.groupby(
1170
+ [DFLT_NAME_COLUMN, DFLT_PNAME_COLUMN, SHAREDMEMBER],
1171
+ as_index=False
1172
+ ).size()
1173
+ duplicated = unique_df[unique_df['size'] > 1]
1174
+ if not duplicated.empty:
1175
+ raise ValueError(
1176
+ f"Duplicated member name for: {duplicated.to_dict(orient='records')}"
1177
+ )
1178
+ else:
1179
+ if df[DFLT_NAME_COLUMN].nunique() != len(df):
1180
+ counter = Counter(df[DFLT_NAME_COLUMN])
1181
+ duplicated = [k for k, v in counter.items() if v > 1]
1182
+ raise ValueError(f"Duplicated member name for: {duplicated}")
1183
+
1184
+
1185
+ class Dimension(AsyncDimension, metaclass=SyncMeta):
1186
+ synchronize = (
1187
+ 'save',
1188
+ 'load_dataframe',
1189
+ 'query',
1190
+ 'to_deepmodel_object',
1191
+ 'sync_data',
1192
+ 'update_parent',
1193
+ 'update_parent_batch',
1194
+ )
1195
+
1196
+ if TYPE_CHECKING:
1197
+ def save(
1198
+ self,
1199
+ reorder: bool = False
1200
+ ): # pragma: no cover
1201
+ ...
1202
+
1203
+ def load_dataframe(
1204
+ self,
1205
+ dataframe: pd.DataFrame,
1206
+ strategy: Union[Strategy, str] = Strategy.incr_replace,
1207
+ reorder: bool = False,
1208
+ **langugage_keys: str
1209
+ ): # pragma: no cover
1210
+ ...
1211
+
1212
+ def query(
1213
+ self,
1214
+ expression: str,
1215
+ fields: Union[List[str], Tuple[str]] = None,
1216
+ as_model: bool = True,
1217
+ role: Union[RoleStrategy, str] = None,
1218
+ role_name: str = None,
1219
+ role_group: str = None,
1220
+ multi_entity_config: List[Dict] = None
1221
+ ) -> Union[
1222
+ List[DimensionMemberBean],
1223
+ List[Dict[str, Any]],
1224
+ List[MultiEntityConfigResult]
1225
+ ]: # pragma: no cover
1226
+ ...
1227
+
1228
+ def to_deepmodel_object(
1229
+ self,
1230
+ object_name: str,
1231
+ expression: str = None,
1232
+ field_map: Dict[str, str] = None,
1233
+ full_replace: bool = False
1234
+ ): # pragma: no cover
1235
+ ...
1236
+
1237
+ def sync_data(self): # pragma: no cover
1238
+ ...
1239
+
1240
+ def update_parent(
1241
+ self,
1242
+ member_name: str,
1243
+ origin_parent: str,
1244
+ new_parent: str,
1245
+ shared_member: bool = False
1246
+ ): # pragma: no cover
1247
+ ...
1248
+
1249
+ def update_parent_batch(
1250
+ self,
1251
+ dataframe: pd.DataFrame
1252
+ ): # pragma: no cover
1253
+ ...
1254
+