deepfos 1.1.60__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. deepfos/__init__.py +6 -0
  2. deepfos/_version.py +21 -0
  3. deepfos/algo/__init__.py +0 -0
  4. deepfos/algo/graph.py +171 -0
  5. deepfos/algo/segtree.py +31 -0
  6. deepfos/api/V1_1/__init__.py +0 -0
  7. deepfos/api/V1_1/business_model.py +119 -0
  8. deepfos/api/V1_1/dimension.py +599 -0
  9. deepfos/api/V1_1/models/__init__.py +0 -0
  10. deepfos/api/V1_1/models/business_model.py +1033 -0
  11. deepfos/api/V1_1/models/dimension.py +2768 -0
  12. deepfos/api/V1_2/__init__.py +0 -0
  13. deepfos/api/V1_2/dimension.py +285 -0
  14. deepfos/api/V1_2/models/__init__.py +0 -0
  15. deepfos/api/V1_2/models/dimension.py +2923 -0
  16. deepfos/api/__init__.py +0 -0
  17. deepfos/api/account.py +167 -0
  18. deepfos/api/accounting_engines.py +147 -0
  19. deepfos/api/app.py +626 -0
  20. deepfos/api/approval_process.py +198 -0
  21. deepfos/api/base.py +983 -0
  22. deepfos/api/business_model.py +160 -0
  23. deepfos/api/consolidation.py +129 -0
  24. deepfos/api/consolidation_process.py +106 -0
  25. deepfos/api/datatable.py +341 -0
  26. deepfos/api/deep_pipeline.py +61 -0
  27. deepfos/api/deepconnector.py +36 -0
  28. deepfos/api/deepfos_task.py +92 -0
  29. deepfos/api/deepmodel.py +188 -0
  30. deepfos/api/dimension.py +486 -0
  31. deepfos/api/financial_model.py +319 -0
  32. deepfos/api/journal_model.py +119 -0
  33. deepfos/api/journal_template.py +132 -0
  34. deepfos/api/memory_financial_model.py +98 -0
  35. deepfos/api/models/__init__.py +3 -0
  36. deepfos/api/models/account.py +483 -0
  37. deepfos/api/models/accounting_engines.py +756 -0
  38. deepfos/api/models/app.py +1338 -0
  39. deepfos/api/models/approval_process.py +1043 -0
  40. deepfos/api/models/base.py +234 -0
  41. deepfos/api/models/business_model.py +805 -0
  42. deepfos/api/models/consolidation.py +711 -0
  43. deepfos/api/models/consolidation_process.py +248 -0
  44. deepfos/api/models/datatable_mysql.py +427 -0
  45. deepfos/api/models/deep_pipeline.py +55 -0
  46. deepfos/api/models/deepconnector.py +28 -0
  47. deepfos/api/models/deepfos_task.py +386 -0
  48. deepfos/api/models/deepmodel.py +308 -0
  49. deepfos/api/models/dimension.py +1576 -0
  50. deepfos/api/models/financial_model.py +1796 -0
  51. deepfos/api/models/journal_model.py +341 -0
  52. deepfos/api/models/journal_template.py +854 -0
  53. deepfos/api/models/memory_financial_model.py +478 -0
  54. deepfos/api/models/platform.py +178 -0
  55. deepfos/api/models/python.py +221 -0
  56. deepfos/api/models/reconciliation_engine.py +411 -0
  57. deepfos/api/models/reconciliation_report.py +161 -0
  58. deepfos/api/models/role_strategy.py +884 -0
  59. deepfos/api/models/smartlist.py +237 -0
  60. deepfos/api/models/space.py +1137 -0
  61. deepfos/api/models/system.py +1065 -0
  62. deepfos/api/models/variable.py +463 -0
  63. deepfos/api/models/workflow.py +946 -0
  64. deepfos/api/platform.py +199 -0
  65. deepfos/api/python.py +90 -0
  66. deepfos/api/reconciliation_engine.py +181 -0
  67. deepfos/api/reconciliation_report.py +64 -0
  68. deepfos/api/role_strategy.py +234 -0
  69. deepfos/api/smartlist.py +69 -0
  70. deepfos/api/space.py +582 -0
  71. deepfos/api/system.py +372 -0
  72. deepfos/api/variable.py +154 -0
  73. deepfos/api/workflow.py +264 -0
  74. deepfos/boost/__init__.py +6 -0
  75. deepfos/boost/py_jstream.py +89 -0
  76. deepfos/boost/py_pandas.py +20 -0
  77. deepfos/cache.py +121 -0
  78. deepfos/config.py +6 -0
  79. deepfos/core/__init__.py +27 -0
  80. deepfos/core/cube/__init__.py +10 -0
  81. deepfos/core/cube/_base.py +462 -0
  82. deepfos/core/cube/constants.py +21 -0
  83. deepfos/core/cube/cube.py +408 -0
  84. deepfos/core/cube/formula.py +707 -0
  85. deepfos/core/cube/syscube.py +532 -0
  86. deepfos/core/cube/typing.py +7 -0
  87. deepfos/core/cube/utils.py +238 -0
  88. deepfos/core/dimension/__init__.py +11 -0
  89. deepfos/core/dimension/_base.py +506 -0
  90. deepfos/core/dimension/dimcreator.py +184 -0
  91. deepfos/core/dimension/dimension.py +472 -0
  92. deepfos/core/dimension/dimexpr.py +271 -0
  93. deepfos/core/dimension/dimmember.py +155 -0
  94. deepfos/core/dimension/eledimension.py +22 -0
  95. deepfos/core/dimension/filters.py +99 -0
  96. deepfos/core/dimension/sysdimension.py +168 -0
  97. deepfos/core/logictable/__init__.py +5 -0
  98. deepfos/core/logictable/_cache.py +141 -0
  99. deepfos/core/logictable/_operator.py +663 -0
  100. deepfos/core/logictable/nodemixin.py +673 -0
  101. deepfos/core/logictable/sqlcondition.py +609 -0
  102. deepfos/core/logictable/tablemodel.py +497 -0
  103. deepfos/db/__init__.py +36 -0
  104. deepfos/db/cipher.py +660 -0
  105. deepfos/db/clickhouse.py +191 -0
  106. deepfos/db/connector.py +195 -0
  107. deepfos/db/daclickhouse.py +171 -0
  108. deepfos/db/dameng.py +101 -0
  109. deepfos/db/damysql.py +189 -0
  110. deepfos/db/dbkits.py +358 -0
  111. deepfos/db/deepengine.py +99 -0
  112. deepfos/db/deepmodel.py +82 -0
  113. deepfos/db/deepmodel_kingbase.py +83 -0
  114. deepfos/db/edb.py +214 -0
  115. deepfos/db/gauss.py +83 -0
  116. deepfos/db/kingbase.py +83 -0
  117. deepfos/db/mysql.py +184 -0
  118. deepfos/db/oracle.py +131 -0
  119. deepfos/db/postgresql.py +192 -0
  120. deepfos/db/sqlserver.py +99 -0
  121. deepfos/db/utils.py +135 -0
  122. deepfos/element/__init__.py +89 -0
  123. deepfos/element/accounting.py +348 -0
  124. deepfos/element/apvlprocess.py +215 -0
  125. deepfos/element/base.py +398 -0
  126. deepfos/element/bizmodel.py +1269 -0
  127. deepfos/element/datatable.py +2467 -0
  128. deepfos/element/deep_pipeline.py +186 -0
  129. deepfos/element/deepconnector.py +59 -0
  130. deepfos/element/deepmodel.py +1806 -0
  131. deepfos/element/dimension.py +1254 -0
  132. deepfos/element/fact_table.py +427 -0
  133. deepfos/element/finmodel.py +1485 -0
  134. deepfos/element/journal.py +840 -0
  135. deepfos/element/journal_template.py +943 -0
  136. deepfos/element/pyscript.py +412 -0
  137. deepfos/element/reconciliation.py +553 -0
  138. deepfos/element/rolestrategy.py +243 -0
  139. deepfos/element/smartlist.py +457 -0
  140. deepfos/element/variable.py +756 -0
  141. deepfos/element/workflow.py +560 -0
  142. deepfos/exceptions/__init__.py +239 -0
  143. deepfos/exceptions/hook.py +86 -0
  144. deepfos/lazy.py +104 -0
  145. deepfos/lazy_import.py +84 -0
  146. deepfos/lib/__init__.py +0 -0
  147. deepfos/lib/_javaobj.py +366 -0
  148. deepfos/lib/asynchronous.py +879 -0
  149. deepfos/lib/concurrency.py +107 -0
  150. deepfos/lib/constant.py +39 -0
  151. deepfos/lib/decorator.py +310 -0
  152. deepfos/lib/deepchart.py +778 -0
  153. deepfos/lib/deepux.py +477 -0
  154. deepfos/lib/discovery.py +273 -0
  155. deepfos/lib/edb_lexer.py +789 -0
  156. deepfos/lib/eureka.py +156 -0
  157. deepfos/lib/filterparser.py +751 -0
  158. deepfos/lib/httpcli.py +106 -0
  159. deepfos/lib/jsonstreamer.py +80 -0
  160. deepfos/lib/msg.py +394 -0
  161. deepfos/lib/nacos.py +225 -0
  162. deepfos/lib/patch.py +92 -0
  163. deepfos/lib/redis.py +241 -0
  164. deepfos/lib/serutils.py +181 -0
  165. deepfos/lib/stopwatch.py +99 -0
  166. deepfos/lib/subtask.py +572 -0
  167. deepfos/lib/sysutils.py +703 -0
  168. deepfos/lib/utils.py +1003 -0
  169. deepfos/local.py +160 -0
  170. deepfos/options.py +670 -0
  171. deepfos/translation.py +237 -0
  172. deepfos-1.1.60.dist-info/METADATA +33 -0
  173. deepfos-1.1.60.dist-info/RECORD +175 -0
  174. deepfos-1.1.60.dist-info/WHEEL +5 -0
  175. deepfos-1.1.60.dist-info/top_level.txt +1 -0
@@ -0,0 +1,751 @@
1
+ import collections
2
+ import re
3
+ from enum import Enum, IntEnum
4
+ from typing import *
5
+ from pydantic import Field, parse_obj_as
6
+
7
+ from deepfos.api.models import BaseModel
8
+ from deepfos.core.logictable.nodemixin import NodeMixin, TreeRenderer
9
+ from deepfos.lib.utils import CIEnum, CIEnumMeta
10
+
11
+
12
+ class ConnectType(Enum):
13
+ and_ = "AND"
14
+ or_ = "OR"
15
+
16
+
17
+ class OperateType(Enum):
18
+ equal = "EQUAL"
19
+ not_equal = "NOT_EQUAL"
20
+ in_list = "IN_LIST"
21
+ not_in_list = "NOT_IN_LIST"
22
+ in_expression = "IN_EXPRESSION"
23
+ contains = "MIDDLE"
24
+ start_with = "START_WITH"
25
+ end_with = "END_WITH"
26
+ is_null = "IS_NULL"
27
+ not_null = "NOT_NULL"
28
+ greater_than = "GREATER_THAN"
29
+ less_than = "LESS_THAN"
30
+ greater_equal = "GREATER_THAN_OR_EQUAL"
31
+ less_equal = "LESS_THAN_OR_EQUAL"
32
+
33
+
34
+ class TimeType(IntEnum, metaclass=CIEnumMeta):
35
+ year = 1
36
+ quarter = 2
37
+ month = 3
38
+ day = 4
39
+
40
+
41
+ class FieldType(CIEnum):
42
+ unset = 'UNSET'
43
+ year = 'YEAR'
44
+ month = 'MONTH'
45
+ day = 'DAY'
46
+ quarter = 'QUARTER'
47
+
48
+ @classmethod
49
+ def is_date(cls, type_):
50
+ return type_ in (cls.year, cls.month, cls.day, cls.quarter)
51
+
52
+
53
+ class ConditionItem(BaseModel):
54
+ operate: OperateType = None
55
+ value: Any = None
56
+
57
+
58
+ class ColumnItem(BaseModel):
59
+ alias: str = None
60
+ columnName: str = None
61
+
62
+
63
+ class SelectColumnItem(ColumnItem):
64
+ conditions: ConditionItem = None
65
+ formatType: FieldType = FieldType.unset
66
+
67
+ @property
68
+ def real_conditions(self):
69
+ return [self.conditions]
70
+
71
+
72
+ class SelectItem(BaseModel):
73
+ connectType: ConnectType = None
74
+ selectGroups: List[SelectColumnItem] = Field(default_factory=list)
75
+
76
+
77
+ class SelectDetail(BaseModel):
78
+ selectedFilter: SelectItem = None
79
+ selectedParam: SelectItem = None
80
+
81
+
82
+ class FilterItem(BaseModel):
83
+ connectType: ConnectType = None
84
+ filterGroups: List['FilterItem'] = Field(default_factory=list)
85
+ alias: str = None
86
+ columnName: str = None
87
+ conditions: List[ConditionItem] = None
88
+ operateType: ConnectType = None
89
+ formatType: FieldType = FieldType.unset
90
+
91
+
92
+ FilterItem.update_forward_refs()
93
+
94
+
95
+ class MeasuresColumnItem(ColumnItem):
96
+ collectMethod: str = None
97
+
98
+
99
+ class TimeColumnItem(ColumnItem):
100
+ formatType: FieldType = None
101
+
102
+
103
+ class QueryInfo(BaseModel):
104
+ selected: SelectDetail = None
105
+ axisRanges: List[FilterItem] = Field(default_factory=list)
106
+ columns: List[ColumnItem] = Field(default_factory=list)
107
+ measures: List[MeasuresColumnItem] = Field(default_factory=list)
108
+ orders: List = Field(default_factory=list)
109
+ timeColumns: List[TimeColumnItem] = Field(default_factory=list)
110
+
111
+
112
+ class Parameter(BaseModel):
113
+ queryInfo: QueryInfo = None
114
+ chartInfo: Any = None
115
+
116
+
117
+ class Sql(str, Enum):
118
+ SQL = 'sql'
119
+ PANDAS_SQL = 'pandas_sql'
120
+
121
+
122
+ def string_replace(s):
123
+ result = s.maketrans('[]', '()')
124
+ value = s.translate(result)
125
+ return value
126
+
127
+
128
+ def _add_func_to_sql(format_type, key, value, operator):
129
+ """为拼接sql条件增加函数"""
130
+ format_type = FieldType[format_type]
131
+ if operator != " in " and operator != " not in ":
132
+ value = f"'{value}'"
133
+ if format_type == FieldType.year:
134
+ return f"CONCAT(YEAR({key})){operator}{value}"
135
+ elif format_type == FieldType.month:
136
+ return f"CONCAT(YEAR({key}),'-',RIGHT(100+MONTH({key}),2)){operator}{value}"
137
+ elif format_type == FieldType.day:
138
+ return f"CONCAT(DATE({key})){operator}{value}"
139
+ elif format_type == FieldType.quarter:
140
+ return f"CONCAT(YEAR({key}),'-Q',QUARTER({key})){operator}{value}"
141
+
142
+
143
+ _RE_DATE_TYPE = re.compile('(?P<year>[0-9]{4})(?:-Q(?P<quarter>[0-9]))?'
144
+ '(?:-0?(?P<month>[0-9]{1,2}))?(?:-0?(?P<day>[0-9]{1,2}))?')
145
+
146
+
147
+ def _equal_with_pandas_dt(field_info: 'FieldNode', value: str, equal=True):
148
+ dt_series = f"`{field_info.name}`.astype('datetime64[ns]').dt"
149
+ precise = field_info.precise
150
+ if equal:
151
+ connect_type, op = 'and', '=='
152
+ else:
153
+ connect_type, op = 'or', '!='
154
+
155
+ year, quarter, month, day = _RE_DATE_TYPE.match(value).groups()
156
+
157
+ query_year = f"{dt_series}.year {op} {year}"
158
+ query_month = f"{dt_series}.month {op} {month}"
159
+ query_day = f"{dt_series}.day {op} {day}"
160
+ # 兼容format date逻辑置后以后
161
+ # 与原逻辑先format date后组织selectGroup中的筛选条件相比
162
+ # 隐含的可能导致selectGroup中的条件中的日期格式
163
+ # 与format date的时间格式不匹配的情况
164
+ if month and day:
165
+ if precise == TimeType.day:
166
+ return f"({query_year} {connect_type} {query_month} {connect_type} {query_day})"
167
+ if precise == TimeType.month:
168
+ return f"({query_year} {connect_type} {query_month})"
169
+ if precise == TimeType.quarter:
170
+ query_quarter = f"{dt_series}.quarter {op} {to_quarter(month)}"
171
+ return f"({query_year} {connect_type} {query_quarter})"
172
+
173
+ if quarter:
174
+ if precise >= TimeType.quarter:
175
+ query_quarter = f"{dt_series}.quarter {op} {quarter}"
176
+ return f"({query_year} {connect_type} {query_quarter})"
177
+
178
+ if month:
179
+ if precise >= TimeType.month:
180
+ return f"({query_year} {connect_type} {query_month})"
181
+ if precise == TimeType.quarter:
182
+ query_quarter = f"{dt_series}.quarter {op} {to_quarter(month)}"
183
+ return f"({query_year} {connect_type} {query_quarter})"
184
+
185
+ return query_year
186
+
187
+
188
+ def to_quarter(month):
189
+ return int((int(month) - 1) / 3) + 1
190
+
191
+
192
+ def _compare_with_pandas_dt(field_info: 'FieldNode', op: str, value: str):
193
+ precise = field_info.precise
194
+ dt_series = f"`{field_info.name}`.astype('datetime64[ns]').dt"
195
+ year, quarter, month, day = _RE_DATE_TYPE.match(value).groups()
196
+ query_year = f"{dt_series}.year {op} {year}"
197
+ # 兼容format date逻辑置后以后
198
+ # 与原逻辑先format date后组织selectGroup中的筛选条件相比
199
+ # 隐含的可能导致selectGroup中的条件中的日期格式
200
+ # 与format date的时间格式不匹配的情况
201
+ if month and day:
202
+ if precise == TimeType.day:
203
+ return f"({dt_series}.year {op[0]} {year} or " \
204
+ f"({dt_series}.year == {year} and {dt_series}.month {op[0]} {month}) or " \
205
+ f"({dt_series}.year == {year} and {dt_series}.month == {month} and {dt_series}.day {op} {day}))"
206
+
207
+ if precise == TimeType.month:
208
+ return f"({dt_series}.year {op[0]} {year} or " \
209
+ f"({dt_series}.year == {year} and {dt_series}.month {op} {month}))"
210
+
211
+ if precise == TimeType.quarter:
212
+ return f"({dt_series}.year {op[0]} {year} or " \
213
+ f"({dt_series}.year == {year} and {dt_series}.quarter {op} {to_quarter(month)}))"
214
+
215
+ if quarter:
216
+ if precise >= TimeType.quarter:
217
+ return f"({dt_series}.year {op[0]} {year} or " \
218
+ f"({dt_series}.year == {year} and {dt_series}.quarter {op} {quarter}))"
219
+
220
+ if month:
221
+ if precise >= TimeType.month:
222
+ return f"({dt_series}.year {op[0]} {year} or " \
223
+ f"({dt_series}.year == {year} and {dt_series}.month {op} {month}))"
224
+
225
+ if precise == TimeType.quarter:
226
+ return f"({dt_series}.year {op[0]} {year} or " \
227
+ f"({dt_series}.year == {year} and {dt_series}.quarter {op} {to_quarter(month)}))"
228
+
229
+ return query_year
230
+
231
+
232
+ dim_members, date_type_fields, dt_precision = {}, {}, {}
233
+
234
+
235
+ def set_dim_members(new_members):
236
+ global dim_members
237
+ dim_members = new_members
238
+
239
+
240
+ def set_date_type_fields(new_fields):
241
+ global date_type_fields
242
+ date_type_fields = new_fields
243
+
244
+
245
+ def set_dt_precision(new_precisions):
246
+ global dt_precision
247
+ dt_precision = new_precisions
248
+
249
+
250
+ def maybe_adapt_real_field_type(column_name, field_type):
251
+ # 兼容2023-02-20迭代前,时间信息取自timeColumns的逻辑
252
+ if field_type == FieldType.unset and column_name in date_type_fields:
253
+ maybe_timetype = date_type_fields[column_name]
254
+ field_type = FieldType[maybe_timetype]
255
+ return field_type
256
+
257
+
258
+ class _BaseOperatorNode(NodeMixin):
259
+ """
260
+ 逻辑操作节点基类
261
+ """
262
+
263
+ def __init__(self, op):
264
+ self.op = op
265
+
266
+ def __str__(self): # pragma: no cover
267
+ return f"<{self.op}>"
268
+
269
+ def get_where_expression(self, sql):
270
+ """
271
+ 获取df.query()条件表达式
272
+ """
273
+ if sql == Sql.SQL:
274
+ return self.op.to_sql()
275
+ if sql == Sql.PANDAS_SQL:
276
+ return self.op.to_pandas_sql()
277
+
278
+ def to_pandas_sql(self):
279
+ """
280
+ 组装不同逻辑操作的条件表达式
281
+ """
282
+ raise NotImplementedError
283
+
284
+
285
+ class LogicalOrOP(_BaseOperatorNode):
286
+ """or 操作"""
287
+
288
+ def to_pandas_sql(self):
289
+ result = ' or '.join(filter(lambda x: x not in ('', '()'),
290
+ [child.to_pandas_sql() for child in self.children]))
291
+ return f"({result})" if result else ''
292
+
293
+ def to_sql(self):
294
+ result = ' or '.join(filter(lambda x: x not in ('', '()'),
295
+ [child.to_sql() for child in self.children]))
296
+ return f"({result})" if result else ''
297
+
298
+
299
+ class LogicalAndOP(_BaseOperatorNode):
300
+ """and 操作"""
301
+
302
+ def to_pandas_sql(self):
303
+ result = ' and '.join(filter(lambda x: x not in ('', '()'),
304
+ [child.to_pandas_sql() for child in self.children]))
305
+ return f"({result})" if result else ''
306
+
307
+ def to_sql(self):
308
+ result = ' and '.join(filter(lambda x: x not in ('', '()'),
309
+ [child.to_sql() for child in self.children]))
310
+ return f"({result})" if result else ''
311
+
312
+
313
+ class LogicalEqualOP(_BaseOperatorNode):
314
+ """==(equal)操作"""
315
+
316
+ def to_pandas_sql(self):
317
+ value = self.children[1].value
318
+ column_name = self.children[0].name
319
+
320
+ if FieldType.is_date(self.children[0].type):
321
+ return _equal_with_pandas_dt(self.children[0], value)
322
+
323
+ if isinstance(value, str):
324
+ return f"`{column_name}` == '{value}'"
325
+
326
+ return f"`{column_name}` == {value}"
327
+
328
+ def to_sql(self):
329
+ column_name = self.children[0].name
330
+ value = string_replace(str(self.children[1].value))
331
+ field_type = self.children[0].type
332
+
333
+ if FieldType.is_date(field_type):
334
+ sql = _add_func_to_sql(field_type, column_name, value, "=")
335
+ return sql
336
+
337
+ if isinstance(self.children[1].value, str):
338
+ return f"{column_name} = '{value}'"
339
+
340
+ return f"{column_name} = {value}"
341
+
342
+
343
+ class LogicalNotEqualOP(_BaseOperatorNode):
344
+ """!=(NotEqual)操作"""
345
+
346
+ def to_pandas_sql(self):
347
+ value = self.children[1].value
348
+ column_name = self.children[0].name
349
+
350
+ if FieldType.is_date(self.children[0].type):
351
+ return _equal_with_pandas_dt(self.children[0], value, False)
352
+
353
+ if isinstance(value, str):
354
+ return f"`{column_name}` != '{value}'"
355
+
356
+ return f"`{column_name}` != {value}"
357
+
358
+ def to_sql(self):
359
+ column_name = self.children[0].name
360
+ value = string_replace(str(self.children[1].value))
361
+ field_type = self.children[0].type
362
+
363
+ if FieldType.is_date(field_type):
364
+ sql = _add_func_to_sql(field_type, column_name, value, "!=")
365
+ return sql
366
+
367
+ if isinstance(self.children[1].value, str):
368
+ return f"{column_name} != '{value}'"
369
+
370
+ return f"{column_name} != {value}"
371
+
372
+
373
+ class LogicalInListOP(_BaseOperatorNode):
374
+ """in(InList) 操作"""
375
+
376
+ def to_pandas_sql(self):
377
+ column_name = self.children[0].name
378
+
379
+ if FieldType.is_date(self.children[0].type):
380
+ filters = []
381
+ for each in self.children[1].value:
382
+ filters.append(_equal_with_pandas_dt(self.children[0], each))
383
+
384
+ return f"({' or '.join(filters)})"
385
+
386
+ return f"`{column_name}` in {self.children[1].value}"
387
+
388
+ def to_sql(self):
389
+ column_name = self.children[0].name
390
+ value = string_replace(str(self.children[1].value))
391
+ field_type = self.children[0].type
392
+
393
+ if FieldType.is_date(field_type):
394
+ sql = _add_func_to_sql(field_type, column_name, value, " in ")
395
+ return sql
396
+
397
+ return f"{self.children[0].name} in {value}"
398
+
399
+
400
+ class LogicalNotInListOP(_BaseOperatorNode):
401
+ """not in(NotInList) 操作"""
402
+
403
+ def to_pandas_sql(self):
404
+ column_name = self.children[0].name
405
+
406
+ if FieldType.is_date(self.children[0].type):
407
+ filters = []
408
+ for each in self.children[1].value:
409
+ filters.append(_equal_with_pandas_dt(self.children[0], each, False))
410
+
411
+ return f"({' and '.join(filters)})"
412
+
413
+ return f"`{column_name}` not in {self.children[1].value}"
414
+
415
+ def to_sql(self):
416
+ column_name = self.children[0].name
417
+ value = string_replace(str(self.children[1].value))
418
+ field_type = self.children[0].type
419
+
420
+ if FieldType.is_date(field_type):
421
+ sql = _add_func_to_sql(field_type, column_name, value, " not in ")
422
+ return sql
423
+
424
+ return f"{self.children[0].name} not in {value}"
425
+
426
+
427
+ class LogicalInExpressionOP(_BaseOperatorNode):
428
+ """维度表达式操作, 经dim query后等同于 in dim members"""
429
+
430
+ def to_pandas_sql(self):
431
+ name = self.children[0].name
432
+ if dim_members and name in dim_members:
433
+ return f"`{name}` in {dim_members[name]}"
434
+
435
+ return ""
436
+
437
+ def to_sql(self):
438
+ name = self.children[0].name
439
+ if dim_members and name in dim_members:
440
+ value = string_replace(str(dim_members[name]))
441
+ return f"{name} in {value}"
442
+
443
+ return ""
444
+
445
+
446
+ class LogicalContainsOP(_BaseOperatorNode):
447
+ """contains 操作:df['col'].str.contains('xxx')"""
448
+
449
+ def to_pandas_sql(self):
450
+ return f"`{self.children[0].name}`.str.contains('{self.children[1].value}')"
451
+
452
+ def to_sql(self):
453
+ return f"{self.children[0].name} like '%{self.children[1].value}%'"
454
+
455
+
456
+ class LogicalStartWithOP(_BaseOperatorNode):
457
+ """StartWith 操作:df['col'].str.startswith('xxx')"""
458
+
459
+ def to_pandas_sql(self):
460
+ return f"`{self.children[0].name}`.str.startswith('{self.children[1].value}')"
461
+
462
+ def to_sql(self):
463
+ return f"{self.children[0].name} like '{self.children[1].value}%'"
464
+
465
+
466
+ class LogicalEndWithOP(_BaseOperatorNode):
467
+ """EndWith 操作:df['col'].str.endswith('xxx')"""
468
+
469
+ def to_pandas_sql(self):
470
+ return f"`{self.children[0].name}`.str.endswith('{self.children[1].value}')"
471
+
472
+ def to_sql(self):
473
+ return f"{self.children[0].name} like '%{self.children[1].value}'"
474
+
475
+
476
+ class LogicalIsNullOP(_BaseOperatorNode):
477
+ """IsNull 操作:df['col'].isnull()"""
478
+
479
+ def to_pandas_sql(self):
480
+ return f"`{self.children[0].name}`.isnull()"
481
+
482
+ def to_sql(self):
483
+ return f"{self.children[0].name} is null"
484
+
485
+
486
+ class LogicalNotNullOP(_BaseOperatorNode):
487
+ """IsNotNull 操作:df['col'].notnull()"""
488
+
489
+ def to_pandas_sql(self):
490
+ return f"`{self.children[0].name}`.notnull()"
491
+
492
+ def to_sql(self):
493
+ return f"{self.children[0].name} is not null"
494
+
495
+
496
+ class LogicalGreaterThanOP(_BaseOperatorNode):
497
+ """> (GREATER_THAN)"""
498
+
499
+ def to_pandas_sql(self):
500
+ value = self.children[1].value
501
+ column_name = self.children[0].name
502
+
503
+ if FieldType.is_date(self.children[0].type):
504
+ return _compare_with_pandas_dt(self.children[0], '>', value)
505
+
506
+ if isinstance(value, str):
507
+ return f"`{column_name}` > '{value}'"
508
+
509
+ return f"`{column_name}` > {value}"
510
+
511
+ def to_sql(self):
512
+ column_name = self.children[0].name
513
+ value = string_replace(str(self.children[1].value))
514
+ field_type = self.children[0].type
515
+
516
+ if FieldType.is_date(field_type):
517
+ sql = _add_func_to_sql(field_type, column_name, value, ">")
518
+ return sql
519
+
520
+ if isinstance(self.children[1].value, str):
521
+ return f"{column_name} > '{value}'"
522
+
523
+ return f"{column_name} > {value}"
524
+
525
+
526
+ class LogicalLessThanOP(_BaseOperatorNode):
527
+ """< (LESS_THAN)"""
528
+
529
+ def to_pandas_sql(self):
530
+ value = self.children[1].value
531
+ column_name = self.children[0].name
532
+
533
+ if FieldType.is_date(self.children[0].type):
534
+ return _compare_with_pandas_dt(self.children[0], '<', value)
535
+
536
+ if isinstance(value, str):
537
+ return f"`{column_name}` < '{value}'"
538
+
539
+ return f"`{column_name}` < {value}"
540
+
541
+ def to_sql(self):
542
+ column_name = self.children[0].name
543
+ value = string_replace(str(self.children[1].value))
544
+ field_type = self.children[0].type
545
+
546
+ if FieldType.is_date(field_type):
547
+ sql = _add_func_to_sql(field_type, column_name, value, "<")
548
+ return sql
549
+
550
+ if isinstance(self.children[1].value, str):
551
+ return f"{column_name} < '{value}'"
552
+
553
+ return f"{column_name} < {value}"
554
+
555
+
556
+ class LogicalGreaterEqualOP(_BaseOperatorNode):
557
+ """>= (GREATER_THAN_OR_EQUAL)"""
558
+
559
+ def to_pandas_sql(self):
560
+ value = self.children[1].value
561
+ column_name = self.children[0].name
562
+
563
+ if FieldType.is_date(self.children[0].type):
564
+ return _compare_with_pandas_dt(self.children[0], '>=', value)
565
+
566
+ if isinstance(value, str):
567
+ return f"`{column_name}` >= '{value}'"
568
+
569
+ return f"`{column_name}` >= {value}"
570
+
571
+ def to_sql(self):
572
+ column_name = self.children[0].name
573
+ value = string_replace(str(self.children[1].value))
574
+ field_type = self.children[0].type
575
+
576
+ if FieldType.is_date(field_type):
577
+ sql = _add_func_to_sql(field_type, column_name, value, ">=")
578
+ return sql
579
+
580
+ if isinstance(self.children[1].value, str):
581
+ return f"{column_name} >= '{value}'"
582
+
583
+ return f"{column_name} >= {value}"
584
+
585
+
586
+ class LogicalLessEqualOP(_BaseOperatorNode):
587
+ """<= (LESS_THAN_OR_EQUAL)"""
588
+
589
+ def to_pandas_sql(self):
590
+ value = self.children[1].value
591
+ column_name = self.children[0].name
592
+
593
+ if FieldType.is_date(self.children[0].type):
594
+ return _compare_with_pandas_dt(self.children[0], '<=', value)
595
+
596
+ if isinstance(value, str):
597
+ return f"`{column_name}` <= '{value}'"
598
+
599
+ return f"`{column_name}` <= {value}"
600
+
601
+ def to_sql(self):
602
+ column_name = self.children[0].name
603
+ value = string_replace(str(self.children[1].value))
604
+ field_type = self.children[0].type
605
+
606
+ if FieldType.is_date(field_type):
607
+ sql = _add_func_to_sql(field_type, column_name, value, "<=")
608
+ return sql
609
+
610
+ if isinstance(self.children[1].value, str):
611
+ return f"{column_name} <= '{value}'"
612
+
613
+ return f"{column_name} <= {value}"
614
+
615
+
616
+ class FieldNode(NodeMixin):
617
+ def __init__(self, name, type_=FieldType.unset, precise=None):
618
+ self.name = name
619
+ self.type = maybe_adapt_real_field_type(name, type_)
620
+ self.precise = precise
621
+
622
+ def __str__(self): # pragma: no cover
623
+ return self.name
624
+
625
+
626
+ class ComparisonVal(NodeMixin):
627
+ def __init__(self, value):
628
+ self.value = value
629
+
630
+ def __str__(self): # pragma: no cover
631
+ return str(self.value)
632
+
633
+
634
+ class LogicalOpFactory:
635
+ """逻辑操作类的匹配工厂"""
636
+ cls_map = {
637
+ ConnectType.or_: LogicalOrOP,
638
+ ConnectType.and_: LogicalAndOP,
639
+ OperateType.equal: LogicalEqualOP,
640
+ OperateType.not_equal: LogicalNotEqualOP,
641
+ OperateType.in_list: LogicalInListOP,
642
+ OperateType.not_in_list: LogicalNotInListOP,
643
+ OperateType.in_expression: LogicalInExpressionOP,
644
+ OperateType.contains: LogicalContainsOP,
645
+ OperateType.start_with: LogicalStartWithOP,
646
+ OperateType.end_with: LogicalEndWithOP,
647
+ OperateType.is_null: LogicalIsNullOP,
648
+ OperateType.not_null: LogicalNotNullOP,
649
+ OperateType.greater_than: LogicalGreaterThanOP,
650
+ OperateType.less_than: LogicalLessThanOP,
651
+ OperateType.greater_equal: LogicalGreaterEqualOP,
652
+ OperateType.less_equal: LogicalLessEqualOP
653
+ }
654
+
655
+ def __new__(cls, value) -> NodeMixin:
656
+ target_cls = cls.cls_map.get(value)
657
+ if target_cls is None: # pragma: no cover
658
+ raise ValueError(f"Unknown Value for python chart: {value}")
659
+ return target_cls(value)
660
+
661
+
662
+ class ASTRoot(NodeMixin):
663
+ pass
664
+
665
+
666
+ class FilterParser:
667
+ """筛选逻辑树解析类"""
668
+ def __init__(
669
+ self,
670
+ source: Union[Dict, FilterItem, SelectItem],
671
+ _all_enable_dim_columns: list = None,
672
+ exclude_dim_expr: bool = False,
673
+ date_round: bool = False
674
+ ):
675
+ if isinstance(source, dict):
676
+ source = parse_obj_as(FilterItem, source)
677
+ self.source = source
678
+ self.root = ASTRoot()
679
+ self._all_enable_dim_columns = _all_enable_dim_columns or []
680
+ self.axis_dim_fields_to_expr = collections.defaultdict(list)
681
+ # 解析时是否去除维度表达式(维度类型字段)
682
+ self.exclude_dim_expr = exclude_dim_expr
683
+ self.date_round = date_round
684
+
685
+ def parse(self):
686
+ self.parse_node(self.source).set_parent(self.root)
687
+
688
+ def parse_node(self, source: Union[FilterItem, SelectItem]) -> NodeMixin:
689
+ if source.connectType:
690
+ logic_node_obj = LogicalOpFactory(source.connectType)
691
+
692
+ if isinstance(source, FilterItem):
693
+ for node_source in source.filterGroups:
694
+ node = self.parse_node(node_source)
695
+ if node:
696
+ node.set_parent(logic_node_obj)
697
+ return logic_node_obj
698
+ else:
699
+ for node_source in source.selectGroups:
700
+ node = self.parse_condition(node_source)
701
+ if node:
702
+ node.set_parent(logic_node_obj)
703
+ return logic_node_obj
704
+ else:
705
+ return self.parse_condition(source)
706
+
707
+ def parse_condition(self, source: Union[FilterItem, SelectColumnItem]) -> Optional[NodeMixin]:
708
+ if self.date_round is False:
709
+ if FieldType.is_date(source.formatType) and (precise := dt_precision.get(source.columnName)):
710
+ if precise < TimeType[source.formatType]:
711
+ raise ValueError('筛选条件包含了低精度日期值与高精度日期值的比较操作\n'
712
+ f'当前数据列名: <{source.columnName}>, 配置精度: {precise.name}')
713
+
714
+ conditions = source.conditions if isinstance(source, FilterItem) else source.real_conditions
715
+ # FilterItem中的维度列特殊处理
716
+ if isinstance(source, FilterItem) and source.columnName in self._all_enable_dim_columns:
717
+ # 维度类型字段只有一个条件
718
+ self.axis_dim_fields_to_expr[source.columnName].append(conditions[0].value)
719
+ # 如果该字段是维度类型且本次解析需要去除维度表达式,则无需解析
720
+ if self.exclude_dim_expr:
721
+ return
722
+
723
+ # 如果有多个条件,使用新的子级逻辑连接
724
+ if len(conditions) > 1:
725
+ logical_op = LogicalOpFactory(source.operateType)
726
+ for cond in conditions:
727
+ comp_op = LogicalOpFactory(cond.operate)
728
+ comp_op.set_parent(logical_op)
729
+ FieldNode(source.columnName, source.formatType, dt_precision.get(source.columnName)).set_parent(comp_op)
730
+ if cond.value is not None:
731
+ ComparisonVal(cond.value).set_parent(comp_op)
732
+ # 如果只有一个条件,不创建新的子级,使用父级的逻辑连接
733
+ else:
734
+ logical_op = LogicalOpFactory(conditions[0].operate)
735
+ FieldNode(source.columnName, source.formatType, dt_precision.get(source.columnName)).set_parent(logical_op)
736
+ if conditions[0].value is not None:
737
+ ComparisonVal(conditions[0].value).set_parent(logical_op)
738
+
739
+ return logical_op
740
+
741
+ def debug(self): # pragma: no cover
742
+ TreeRenderer().show(self.root)
743
+
744
+ def make_query(self, dim_mbrs=None, date_fields=None, sql: Enum = Sql.PANDAS_SQL) -> str:
745
+ query = _BaseOperatorNode(self.root.children[0])
746
+ global dim_members, date_type_fields
747
+ if dim_mbrs is not None:
748
+ dim_members = dim_mbrs
749
+ if date_fields is not None:
750
+ date_type_fields = date_fields
751
+ return query.get_where_expression(sql)