deepfos 1.1.60__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deepfos/__init__.py +6 -0
- deepfos/_version.py +21 -0
- deepfos/algo/__init__.py +0 -0
- deepfos/algo/graph.py +171 -0
- deepfos/algo/segtree.py +31 -0
- deepfos/api/V1_1/__init__.py +0 -0
- deepfos/api/V1_1/business_model.py +119 -0
- deepfos/api/V1_1/dimension.py +599 -0
- deepfos/api/V1_1/models/__init__.py +0 -0
- deepfos/api/V1_1/models/business_model.py +1033 -0
- deepfos/api/V1_1/models/dimension.py +2768 -0
- deepfos/api/V1_2/__init__.py +0 -0
- deepfos/api/V1_2/dimension.py +285 -0
- deepfos/api/V1_2/models/__init__.py +0 -0
- deepfos/api/V1_2/models/dimension.py +2923 -0
- deepfos/api/__init__.py +0 -0
- deepfos/api/account.py +167 -0
- deepfos/api/accounting_engines.py +147 -0
- deepfos/api/app.py +626 -0
- deepfos/api/approval_process.py +198 -0
- deepfos/api/base.py +983 -0
- deepfos/api/business_model.py +160 -0
- deepfos/api/consolidation.py +129 -0
- deepfos/api/consolidation_process.py +106 -0
- deepfos/api/datatable.py +341 -0
- deepfos/api/deep_pipeline.py +61 -0
- deepfos/api/deepconnector.py +36 -0
- deepfos/api/deepfos_task.py +92 -0
- deepfos/api/deepmodel.py +188 -0
- deepfos/api/dimension.py +486 -0
- deepfos/api/financial_model.py +319 -0
- deepfos/api/journal_model.py +119 -0
- deepfos/api/journal_template.py +132 -0
- deepfos/api/memory_financial_model.py +98 -0
- deepfos/api/models/__init__.py +3 -0
- deepfos/api/models/account.py +483 -0
- deepfos/api/models/accounting_engines.py +756 -0
- deepfos/api/models/app.py +1338 -0
- deepfos/api/models/approval_process.py +1043 -0
- deepfos/api/models/base.py +234 -0
- deepfos/api/models/business_model.py +805 -0
- deepfos/api/models/consolidation.py +711 -0
- deepfos/api/models/consolidation_process.py +248 -0
- deepfos/api/models/datatable_mysql.py +427 -0
- deepfos/api/models/deep_pipeline.py +55 -0
- deepfos/api/models/deepconnector.py +28 -0
- deepfos/api/models/deepfos_task.py +386 -0
- deepfos/api/models/deepmodel.py +308 -0
- deepfos/api/models/dimension.py +1576 -0
- deepfos/api/models/financial_model.py +1796 -0
- deepfos/api/models/journal_model.py +341 -0
- deepfos/api/models/journal_template.py +854 -0
- deepfos/api/models/memory_financial_model.py +478 -0
- deepfos/api/models/platform.py +178 -0
- deepfos/api/models/python.py +221 -0
- deepfos/api/models/reconciliation_engine.py +411 -0
- deepfos/api/models/reconciliation_report.py +161 -0
- deepfos/api/models/role_strategy.py +884 -0
- deepfos/api/models/smartlist.py +237 -0
- deepfos/api/models/space.py +1137 -0
- deepfos/api/models/system.py +1065 -0
- deepfos/api/models/variable.py +463 -0
- deepfos/api/models/workflow.py +946 -0
- deepfos/api/platform.py +199 -0
- deepfos/api/python.py +90 -0
- deepfos/api/reconciliation_engine.py +181 -0
- deepfos/api/reconciliation_report.py +64 -0
- deepfos/api/role_strategy.py +234 -0
- deepfos/api/smartlist.py +69 -0
- deepfos/api/space.py +582 -0
- deepfos/api/system.py +372 -0
- deepfos/api/variable.py +154 -0
- deepfos/api/workflow.py +264 -0
- deepfos/boost/__init__.py +6 -0
- deepfos/boost/py_jstream.py +89 -0
- deepfos/boost/py_pandas.py +20 -0
- deepfos/cache.py +121 -0
- deepfos/config.py +6 -0
- deepfos/core/__init__.py +27 -0
- deepfos/core/cube/__init__.py +10 -0
- deepfos/core/cube/_base.py +462 -0
- deepfos/core/cube/constants.py +21 -0
- deepfos/core/cube/cube.py +408 -0
- deepfos/core/cube/formula.py +707 -0
- deepfos/core/cube/syscube.py +532 -0
- deepfos/core/cube/typing.py +7 -0
- deepfos/core/cube/utils.py +238 -0
- deepfos/core/dimension/__init__.py +11 -0
- deepfos/core/dimension/_base.py +506 -0
- deepfos/core/dimension/dimcreator.py +184 -0
- deepfos/core/dimension/dimension.py +472 -0
- deepfos/core/dimension/dimexpr.py +271 -0
- deepfos/core/dimension/dimmember.py +155 -0
- deepfos/core/dimension/eledimension.py +22 -0
- deepfos/core/dimension/filters.py +99 -0
- deepfos/core/dimension/sysdimension.py +168 -0
- deepfos/core/logictable/__init__.py +5 -0
- deepfos/core/logictable/_cache.py +141 -0
- deepfos/core/logictable/_operator.py +663 -0
- deepfos/core/logictable/nodemixin.py +673 -0
- deepfos/core/logictable/sqlcondition.py +609 -0
- deepfos/core/logictable/tablemodel.py +497 -0
- deepfos/db/__init__.py +36 -0
- deepfos/db/cipher.py +660 -0
- deepfos/db/clickhouse.py +191 -0
- deepfos/db/connector.py +195 -0
- deepfos/db/daclickhouse.py +171 -0
- deepfos/db/dameng.py +101 -0
- deepfos/db/damysql.py +189 -0
- deepfos/db/dbkits.py +358 -0
- deepfos/db/deepengine.py +99 -0
- deepfos/db/deepmodel.py +82 -0
- deepfos/db/deepmodel_kingbase.py +83 -0
- deepfos/db/edb.py +214 -0
- deepfos/db/gauss.py +83 -0
- deepfos/db/kingbase.py +83 -0
- deepfos/db/mysql.py +184 -0
- deepfos/db/oracle.py +131 -0
- deepfos/db/postgresql.py +192 -0
- deepfos/db/sqlserver.py +99 -0
- deepfos/db/utils.py +135 -0
- deepfos/element/__init__.py +89 -0
- deepfos/element/accounting.py +348 -0
- deepfos/element/apvlprocess.py +215 -0
- deepfos/element/base.py +398 -0
- deepfos/element/bizmodel.py +1269 -0
- deepfos/element/datatable.py +2467 -0
- deepfos/element/deep_pipeline.py +186 -0
- deepfos/element/deepconnector.py +59 -0
- deepfos/element/deepmodel.py +1806 -0
- deepfos/element/dimension.py +1254 -0
- deepfos/element/fact_table.py +427 -0
- deepfos/element/finmodel.py +1485 -0
- deepfos/element/journal.py +840 -0
- deepfos/element/journal_template.py +943 -0
- deepfos/element/pyscript.py +412 -0
- deepfos/element/reconciliation.py +553 -0
- deepfos/element/rolestrategy.py +243 -0
- deepfos/element/smartlist.py +457 -0
- deepfos/element/variable.py +756 -0
- deepfos/element/workflow.py +560 -0
- deepfos/exceptions/__init__.py +239 -0
- deepfos/exceptions/hook.py +86 -0
- deepfos/lazy.py +104 -0
- deepfos/lazy_import.py +84 -0
- deepfos/lib/__init__.py +0 -0
- deepfos/lib/_javaobj.py +366 -0
- deepfos/lib/asynchronous.py +879 -0
- deepfos/lib/concurrency.py +107 -0
- deepfos/lib/constant.py +39 -0
- deepfos/lib/decorator.py +310 -0
- deepfos/lib/deepchart.py +778 -0
- deepfos/lib/deepux.py +477 -0
- deepfos/lib/discovery.py +273 -0
- deepfos/lib/edb_lexer.py +789 -0
- deepfos/lib/eureka.py +156 -0
- deepfos/lib/filterparser.py +751 -0
- deepfos/lib/httpcli.py +106 -0
- deepfos/lib/jsonstreamer.py +80 -0
- deepfos/lib/msg.py +394 -0
- deepfos/lib/nacos.py +225 -0
- deepfos/lib/patch.py +92 -0
- deepfos/lib/redis.py +241 -0
- deepfos/lib/serutils.py +181 -0
- deepfos/lib/stopwatch.py +99 -0
- deepfos/lib/subtask.py +572 -0
- deepfos/lib/sysutils.py +703 -0
- deepfos/lib/utils.py +1003 -0
- deepfos/local.py +160 -0
- deepfos/options.py +670 -0
- deepfos/translation.py +237 -0
- deepfos-1.1.60.dist-info/METADATA +33 -0
- deepfos-1.1.60.dist-info/RECORD +175 -0
- deepfos-1.1.60.dist-info/WHEEL +5 -0
- deepfos-1.1.60.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,751 @@
|
|
|
1
|
+
import collections
|
|
2
|
+
import re
|
|
3
|
+
from enum import Enum, IntEnum
|
|
4
|
+
from typing import *
|
|
5
|
+
from pydantic import Field, parse_obj_as
|
|
6
|
+
|
|
7
|
+
from deepfos.api.models import BaseModel
|
|
8
|
+
from deepfos.core.logictable.nodemixin import NodeMixin, TreeRenderer
|
|
9
|
+
from deepfos.lib.utils import CIEnum, CIEnumMeta
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ConnectType(Enum):
|
|
13
|
+
and_ = "AND"
|
|
14
|
+
or_ = "OR"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class OperateType(Enum):
|
|
18
|
+
equal = "EQUAL"
|
|
19
|
+
not_equal = "NOT_EQUAL"
|
|
20
|
+
in_list = "IN_LIST"
|
|
21
|
+
not_in_list = "NOT_IN_LIST"
|
|
22
|
+
in_expression = "IN_EXPRESSION"
|
|
23
|
+
contains = "MIDDLE"
|
|
24
|
+
start_with = "START_WITH"
|
|
25
|
+
end_with = "END_WITH"
|
|
26
|
+
is_null = "IS_NULL"
|
|
27
|
+
not_null = "NOT_NULL"
|
|
28
|
+
greater_than = "GREATER_THAN"
|
|
29
|
+
less_than = "LESS_THAN"
|
|
30
|
+
greater_equal = "GREATER_THAN_OR_EQUAL"
|
|
31
|
+
less_equal = "LESS_THAN_OR_EQUAL"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class TimeType(IntEnum, metaclass=CIEnumMeta):
|
|
35
|
+
year = 1
|
|
36
|
+
quarter = 2
|
|
37
|
+
month = 3
|
|
38
|
+
day = 4
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class FieldType(CIEnum):
|
|
42
|
+
unset = 'UNSET'
|
|
43
|
+
year = 'YEAR'
|
|
44
|
+
month = 'MONTH'
|
|
45
|
+
day = 'DAY'
|
|
46
|
+
quarter = 'QUARTER'
|
|
47
|
+
|
|
48
|
+
@classmethod
|
|
49
|
+
def is_date(cls, type_):
|
|
50
|
+
return type_ in (cls.year, cls.month, cls.day, cls.quarter)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class ConditionItem(BaseModel):
|
|
54
|
+
operate: OperateType = None
|
|
55
|
+
value: Any = None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class ColumnItem(BaseModel):
|
|
59
|
+
alias: str = None
|
|
60
|
+
columnName: str = None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class SelectColumnItem(ColumnItem):
|
|
64
|
+
conditions: ConditionItem = None
|
|
65
|
+
formatType: FieldType = FieldType.unset
|
|
66
|
+
|
|
67
|
+
@property
|
|
68
|
+
def real_conditions(self):
|
|
69
|
+
return [self.conditions]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class SelectItem(BaseModel):
|
|
73
|
+
connectType: ConnectType = None
|
|
74
|
+
selectGroups: List[SelectColumnItem] = Field(default_factory=list)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class SelectDetail(BaseModel):
|
|
78
|
+
selectedFilter: SelectItem = None
|
|
79
|
+
selectedParam: SelectItem = None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class FilterItem(BaseModel):
|
|
83
|
+
connectType: ConnectType = None
|
|
84
|
+
filterGroups: List['FilterItem'] = Field(default_factory=list)
|
|
85
|
+
alias: str = None
|
|
86
|
+
columnName: str = None
|
|
87
|
+
conditions: List[ConditionItem] = None
|
|
88
|
+
operateType: ConnectType = None
|
|
89
|
+
formatType: FieldType = FieldType.unset
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
FilterItem.update_forward_refs()
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class MeasuresColumnItem(ColumnItem):
|
|
96
|
+
collectMethod: str = None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class TimeColumnItem(ColumnItem):
|
|
100
|
+
formatType: FieldType = None
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class QueryInfo(BaseModel):
|
|
104
|
+
selected: SelectDetail = None
|
|
105
|
+
axisRanges: List[FilterItem] = Field(default_factory=list)
|
|
106
|
+
columns: List[ColumnItem] = Field(default_factory=list)
|
|
107
|
+
measures: List[MeasuresColumnItem] = Field(default_factory=list)
|
|
108
|
+
orders: List = Field(default_factory=list)
|
|
109
|
+
timeColumns: List[TimeColumnItem] = Field(default_factory=list)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class Parameter(BaseModel):
|
|
113
|
+
queryInfo: QueryInfo = None
|
|
114
|
+
chartInfo: Any = None
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class Sql(str, Enum):
|
|
118
|
+
SQL = 'sql'
|
|
119
|
+
PANDAS_SQL = 'pandas_sql'
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def string_replace(s):
|
|
123
|
+
result = s.maketrans('[]', '()')
|
|
124
|
+
value = s.translate(result)
|
|
125
|
+
return value
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _add_func_to_sql(format_type, key, value, operator):
|
|
129
|
+
"""为拼接sql条件增加函数"""
|
|
130
|
+
format_type = FieldType[format_type]
|
|
131
|
+
if operator != " in " and operator != " not in ":
|
|
132
|
+
value = f"'{value}'"
|
|
133
|
+
if format_type == FieldType.year:
|
|
134
|
+
return f"CONCAT(YEAR({key})){operator}{value}"
|
|
135
|
+
elif format_type == FieldType.month:
|
|
136
|
+
return f"CONCAT(YEAR({key}),'-',RIGHT(100+MONTH({key}),2)){operator}{value}"
|
|
137
|
+
elif format_type == FieldType.day:
|
|
138
|
+
return f"CONCAT(DATE({key})){operator}{value}"
|
|
139
|
+
elif format_type == FieldType.quarter:
|
|
140
|
+
return f"CONCAT(YEAR({key}),'-Q',QUARTER({key})){operator}{value}"
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
_RE_DATE_TYPE = re.compile('(?P<year>[0-9]{4})(?:-Q(?P<quarter>[0-9]))?'
|
|
144
|
+
'(?:-0?(?P<month>[0-9]{1,2}))?(?:-0?(?P<day>[0-9]{1,2}))?')
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _equal_with_pandas_dt(field_info: 'FieldNode', value: str, equal=True):
|
|
148
|
+
dt_series = f"`{field_info.name}`.astype('datetime64[ns]').dt"
|
|
149
|
+
precise = field_info.precise
|
|
150
|
+
if equal:
|
|
151
|
+
connect_type, op = 'and', '=='
|
|
152
|
+
else:
|
|
153
|
+
connect_type, op = 'or', '!='
|
|
154
|
+
|
|
155
|
+
year, quarter, month, day = _RE_DATE_TYPE.match(value).groups()
|
|
156
|
+
|
|
157
|
+
query_year = f"{dt_series}.year {op} {year}"
|
|
158
|
+
query_month = f"{dt_series}.month {op} {month}"
|
|
159
|
+
query_day = f"{dt_series}.day {op} {day}"
|
|
160
|
+
# 兼容format date逻辑置后以后
|
|
161
|
+
# 与原逻辑先format date后组织selectGroup中的筛选条件相比
|
|
162
|
+
# 隐含的可能导致selectGroup中的条件中的日期格式
|
|
163
|
+
# 与format date的时间格式不匹配的情况
|
|
164
|
+
if month and day:
|
|
165
|
+
if precise == TimeType.day:
|
|
166
|
+
return f"({query_year} {connect_type} {query_month} {connect_type} {query_day})"
|
|
167
|
+
if precise == TimeType.month:
|
|
168
|
+
return f"({query_year} {connect_type} {query_month})"
|
|
169
|
+
if precise == TimeType.quarter:
|
|
170
|
+
query_quarter = f"{dt_series}.quarter {op} {to_quarter(month)}"
|
|
171
|
+
return f"({query_year} {connect_type} {query_quarter})"
|
|
172
|
+
|
|
173
|
+
if quarter:
|
|
174
|
+
if precise >= TimeType.quarter:
|
|
175
|
+
query_quarter = f"{dt_series}.quarter {op} {quarter}"
|
|
176
|
+
return f"({query_year} {connect_type} {query_quarter})"
|
|
177
|
+
|
|
178
|
+
if month:
|
|
179
|
+
if precise >= TimeType.month:
|
|
180
|
+
return f"({query_year} {connect_type} {query_month})"
|
|
181
|
+
if precise == TimeType.quarter:
|
|
182
|
+
query_quarter = f"{dt_series}.quarter {op} {to_quarter(month)}"
|
|
183
|
+
return f"({query_year} {connect_type} {query_quarter})"
|
|
184
|
+
|
|
185
|
+
return query_year
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def to_quarter(month):
|
|
189
|
+
return int((int(month) - 1) / 3) + 1
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _compare_with_pandas_dt(field_info: 'FieldNode', op: str, value: str):
|
|
193
|
+
precise = field_info.precise
|
|
194
|
+
dt_series = f"`{field_info.name}`.astype('datetime64[ns]').dt"
|
|
195
|
+
year, quarter, month, day = _RE_DATE_TYPE.match(value).groups()
|
|
196
|
+
query_year = f"{dt_series}.year {op} {year}"
|
|
197
|
+
# 兼容format date逻辑置后以后
|
|
198
|
+
# 与原逻辑先format date后组织selectGroup中的筛选条件相比
|
|
199
|
+
# 隐含的可能导致selectGroup中的条件中的日期格式
|
|
200
|
+
# 与format date的时间格式不匹配的情况
|
|
201
|
+
if month and day:
|
|
202
|
+
if precise == TimeType.day:
|
|
203
|
+
return f"({dt_series}.year {op[0]} {year} or " \
|
|
204
|
+
f"({dt_series}.year == {year} and {dt_series}.month {op[0]} {month}) or " \
|
|
205
|
+
f"({dt_series}.year == {year} and {dt_series}.month == {month} and {dt_series}.day {op} {day}))"
|
|
206
|
+
|
|
207
|
+
if precise == TimeType.month:
|
|
208
|
+
return f"({dt_series}.year {op[0]} {year} or " \
|
|
209
|
+
f"({dt_series}.year == {year} and {dt_series}.month {op} {month}))"
|
|
210
|
+
|
|
211
|
+
if precise == TimeType.quarter:
|
|
212
|
+
return f"({dt_series}.year {op[0]} {year} or " \
|
|
213
|
+
f"({dt_series}.year == {year} and {dt_series}.quarter {op} {to_quarter(month)}))"
|
|
214
|
+
|
|
215
|
+
if quarter:
|
|
216
|
+
if precise >= TimeType.quarter:
|
|
217
|
+
return f"({dt_series}.year {op[0]} {year} or " \
|
|
218
|
+
f"({dt_series}.year == {year} and {dt_series}.quarter {op} {quarter}))"
|
|
219
|
+
|
|
220
|
+
if month:
|
|
221
|
+
if precise >= TimeType.month:
|
|
222
|
+
return f"({dt_series}.year {op[0]} {year} or " \
|
|
223
|
+
f"({dt_series}.year == {year} and {dt_series}.month {op} {month}))"
|
|
224
|
+
|
|
225
|
+
if precise == TimeType.quarter:
|
|
226
|
+
return f"({dt_series}.year {op[0]} {year} or " \
|
|
227
|
+
f"({dt_series}.year == {year} and {dt_series}.quarter {op} {to_quarter(month)}))"
|
|
228
|
+
|
|
229
|
+
return query_year
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
dim_members, date_type_fields, dt_precision = {}, {}, {}
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def set_dim_members(new_members):
|
|
236
|
+
global dim_members
|
|
237
|
+
dim_members = new_members
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def set_date_type_fields(new_fields):
|
|
241
|
+
global date_type_fields
|
|
242
|
+
date_type_fields = new_fields
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def set_dt_precision(new_precisions):
|
|
246
|
+
global dt_precision
|
|
247
|
+
dt_precision = new_precisions
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def maybe_adapt_real_field_type(column_name, field_type):
|
|
251
|
+
# 兼容2023-02-20迭代前,时间信息取自timeColumns的逻辑
|
|
252
|
+
if field_type == FieldType.unset and column_name in date_type_fields:
|
|
253
|
+
maybe_timetype = date_type_fields[column_name]
|
|
254
|
+
field_type = FieldType[maybe_timetype]
|
|
255
|
+
return field_type
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
class _BaseOperatorNode(NodeMixin):
|
|
259
|
+
"""
|
|
260
|
+
逻辑操作节点基类
|
|
261
|
+
"""
|
|
262
|
+
|
|
263
|
+
def __init__(self, op):
|
|
264
|
+
self.op = op
|
|
265
|
+
|
|
266
|
+
def __str__(self): # pragma: no cover
|
|
267
|
+
return f"<{self.op}>"
|
|
268
|
+
|
|
269
|
+
def get_where_expression(self, sql):
|
|
270
|
+
"""
|
|
271
|
+
获取df.query()条件表达式
|
|
272
|
+
"""
|
|
273
|
+
if sql == Sql.SQL:
|
|
274
|
+
return self.op.to_sql()
|
|
275
|
+
if sql == Sql.PANDAS_SQL:
|
|
276
|
+
return self.op.to_pandas_sql()
|
|
277
|
+
|
|
278
|
+
def to_pandas_sql(self):
|
|
279
|
+
"""
|
|
280
|
+
组装不同逻辑操作的条件表达式
|
|
281
|
+
"""
|
|
282
|
+
raise NotImplementedError
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
class LogicalOrOP(_BaseOperatorNode):
|
|
286
|
+
"""or 操作"""
|
|
287
|
+
|
|
288
|
+
def to_pandas_sql(self):
|
|
289
|
+
result = ' or '.join(filter(lambda x: x not in ('', '()'),
|
|
290
|
+
[child.to_pandas_sql() for child in self.children]))
|
|
291
|
+
return f"({result})" if result else ''
|
|
292
|
+
|
|
293
|
+
def to_sql(self):
|
|
294
|
+
result = ' or '.join(filter(lambda x: x not in ('', '()'),
|
|
295
|
+
[child.to_sql() for child in self.children]))
|
|
296
|
+
return f"({result})" if result else ''
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
class LogicalAndOP(_BaseOperatorNode):
|
|
300
|
+
"""and 操作"""
|
|
301
|
+
|
|
302
|
+
def to_pandas_sql(self):
|
|
303
|
+
result = ' and '.join(filter(lambda x: x not in ('', '()'),
|
|
304
|
+
[child.to_pandas_sql() for child in self.children]))
|
|
305
|
+
return f"({result})" if result else ''
|
|
306
|
+
|
|
307
|
+
def to_sql(self):
|
|
308
|
+
result = ' and '.join(filter(lambda x: x not in ('', '()'),
|
|
309
|
+
[child.to_sql() for child in self.children]))
|
|
310
|
+
return f"({result})" if result else ''
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
class LogicalEqualOP(_BaseOperatorNode):
|
|
314
|
+
"""==(equal)操作"""
|
|
315
|
+
|
|
316
|
+
def to_pandas_sql(self):
|
|
317
|
+
value = self.children[1].value
|
|
318
|
+
column_name = self.children[0].name
|
|
319
|
+
|
|
320
|
+
if FieldType.is_date(self.children[0].type):
|
|
321
|
+
return _equal_with_pandas_dt(self.children[0], value)
|
|
322
|
+
|
|
323
|
+
if isinstance(value, str):
|
|
324
|
+
return f"`{column_name}` == '{value}'"
|
|
325
|
+
|
|
326
|
+
return f"`{column_name}` == {value}"
|
|
327
|
+
|
|
328
|
+
def to_sql(self):
|
|
329
|
+
column_name = self.children[0].name
|
|
330
|
+
value = string_replace(str(self.children[1].value))
|
|
331
|
+
field_type = self.children[0].type
|
|
332
|
+
|
|
333
|
+
if FieldType.is_date(field_type):
|
|
334
|
+
sql = _add_func_to_sql(field_type, column_name, value, "=")
|
|
335
|
+
return sql
|
|
336
|
+
|
|
337
|
+
if isinstance(self.children[1].value, str):
|
|
338
|
+
return f"{column_name} = '{value}'"
|
|
339
|
+
|
|
340
|
+
return f"{column_name} = {value}"
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
class LogicalNotEqualOP(_BaseOperatorNode):
|
|
344
|
+
"""!=(NotEqual)操作"""
|
|
345
|
+
|
|
346
|
+
def to_pandas_sql(self):
|
|
347
|
+
value = self.children[1].value
|
|
348
|
+
column_name = self.children[0].name
|
|
349
|
+
|
|
350
|
+
if FieldType.is_date(self.children[0].type):
|
|
351
|
+
return _equal_with_pandas_dt(self.children[0], value, False)
|
|
352
|
+
|
|
353
|
+
if isinstance(value, str):
|
|
354
|
+
return f"`{column_name}` != '{value}'"
|
|
355
|
+
|
|
356
|
+
return f"`{column_name}` != {value}"
|
|
357
|
+
|
|
358
|
+
def to_sql(self):
|
|
359
|
+
column_name = self.children[0].name
|
|
360
|
+
value = string_replace(str(self.children[1].value))
|
|
361
|
+
field_type = self.children[0].type
|
|
362
|
+
|
|
363
|
+
if FieldType.is_date(field_type):
|
|
364
|
+
sql = _add_func_to_sql(field_type, column_name, value, "!=")
|
|
365
|
+
return sql
|
|
366
|
+
|
|
367
|
+
if isinstance(self.children[1].value, str):
|
|
368
|
+
return f"{column_name} != '{value}'"
|
|
369
|
+
|
|
370
|
+
return f"{column_name} != {value}"
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
class LogicalInListOP(_BaseOperatorNode):
|
|
374
|
+
"""in(InList) 操作"""
|
|
375
|
+
|
|
376
|
+
def to_pandas_sql(self):
|
|
377
|
+
column_name = self.children[0].name
|
|
378
|
+
|
|
379
|
+
if FieldType.is_date(self.children[0].type):
|
|
380
|
+
filters = []
|
|
381
|
+
for each in self.children[1].value:
|
|
382
|
+
filters.append(_equal_with_pandas_dt(self.children[0], each))
|
|
383
|
+
|
|
384
|
+
return f"({' or '.join(filters)})"
|
|
385
|
+
|
|
386
|
+
return f"`{column_name}` in {self.children[1].value}"
|
|
387
|
+
|
|
388
|
+
def to_sql(self):
|
|
389
|
+
column_name = self.children[0].name
|
|
390
|
+
value = string_replace(str(self.children[1].value))
|
|
391
|
+
field_type = self.children[0].type
|
|
392
|
+
|
|
393
|
+
if FieldType.is_date(field_type):
|
|
394
|
+
sql = _add_func_to_sql(field_type, column_name, value, " in ")
|
|
395
|
+
return sql
|
|
396
|
+
|
|
397
|
+
return f"{self.children[0].name} in {value}"
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
class LogicalNotInListOP(_BaseOperatorNode):
|
|
401
|
+
"""not in(NotInList) 操作"""
|
|
402
|
+
|
|
403
|
+
def to_pandas_sql(self):
|
|
404
|
+
column_name = self.children[0].name
|
|
405
|
+
|
|
406
|
+
if FieldType.is_date(self.children[0].type):
|
|
407
|
+
filters = []
|
|
408
|
+
for each in self.children[1].value:
|
|
409
|
+
filters.append(_equal_with_pandas_dt(self.children[0], each, False))
|
|
410
|
+
|
|
411
|
+
return f"({' and '.join(filters)})"
|
|
412
|
+
|
|
413
|
+
return f"`{column_name}` not in {self.children[1].value}"
|
|
414
|
+
|
|
415
|
+
def to_sql(self):
|
|
416
|
+
column_name = self.children[0].name
|
|
417
|
+
value = string_replace(str(self.children[1].value))
|
|
418
|
+
field_type = self.children[0].type
|
|
419
|
+
|
|
420
|
+
if FieldType.is_date(field_type):
|
|
421
|
+
sql = _add_func_to_sql(field_type, column_name, value, " not in ")
|
|
422
|
+
return sql
|
|
423
|
+
|
|
424
|
+
return f"{self.children[0].name} not in {value}"
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
class LogicalInExpressionOP(_BaseOperatorNode):
|
|
428
|
+
"""维度表达式操作, 经dim query后等同于 in dim members"""
|
|
429
|
+
|
|
430
|
+
def to_pandas_sql(self):
|
|
431
|
+
name = self.children[0].name
|
|
432
|
+
if dim_members and name in dim_members:
|
|
433
|
+
return f"`{name}` in {dim_members[name]}"
|
|
434
|
+
|
|
435
|
+
return ""
|
|
436
|
+
|
|
437
|
+
def to_sql(self):
|
|
438
|
+
name = self.children[0].name
|
|
439
|
+
if dim_members and name in dim_members:
|
|
440
|
+
value = string_replace(str(dim_members[name]))
|
|
441
|
+
return f"{name} in {value}"
|
|
442
|
+
|
|
443
|
+
return ""
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
class LogicalContainsOP(_BaseOperatorNode):
|
|
447
|
+
"""contains 操作:df['col'].str.contains('xxx')"""
|
|
448
|
+
|
|
449
|
+
def to_pandas_sql(self):
|
|
450
|
+
return f"`{self.children[0].name}`.str.contains('{self.children[1].value}')"
|
|
451
|
+
|
|
452
|
+
def to_sql(self):
|
|
453
|
+
return f"{self.children[0].name} like '%{self.children[1].value}%'"
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
class LogicalStartWithOP(_BaseOperatorNode):
|
|
457
|
+
"""StartWith 操作:df['col'].str.startswith('xxx')"""
|
|
458
|
+
|
|
459
|
+
def to_pandas_sql(self):
|
|
460
|
+
return f"`{self.children[0].name}`.str.startswith('{self.children[1].value}')"
|
|
461
|
+
|
|
462
|
+
def to_sql(self):
|
|
463
|
+
return f"{self.children[0].name} like '{self.children[1].value}%'"
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
class LogicalEndWithOP(_BaseOperatorNode):
|
|
467
|
+
"""EndWith 操作:df['col'].str.endswith('xxx')"""
|
|
468
|
+
|
|
469
|
+
def to_pandas_sql(self):
|
|
470
|
+
return f"`{self.children[0].name}`.str.endswith('{self.children[1].value}')"
|
|
471
|
+
|
|
472
|
+
def to_sql(self):
|
|
473
|
+
return f"{self.children[0].name} like '%{self.children[1].value}'"
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
class LogicalIsNullOP(_BaseOperatorNode):
|
|
477
|
+
"""IsNull 操作:df['col'].isnull()"""
|
|
478
|
+
|
|
479
|
+
def to_pandas_sql(self):
|
|
480
|
+
return f"`{self.children[0].name}`.isnull()"
|
|
481
|
+
|
|
482
|
+
def to_sql(self):
|
|
483
|
+
return f"{self.children[0].name} is null"
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
class LogicalNotNullOP(_BaseOperatorNode):
|
|
487
|
+
"""IsNotNull 操作:df['col'].notnull()"""
|
|
488
|
+
|
|
489
|
+
def to_pandas_sql(self):
|
|
490
|
+
return f"`{self.children[0].name}`.notnull()"
|
|
491
|
+
|
|
492
|
+
def to_sql(self):
|
|
493
|
+
return f"{self.children[0].name} is not null"
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
class LogicalGreaterThanOP(_BaseOperatorNode):
|
|
497
|
+
"""> (GREATER_THAN)"""
|
|
498
|
+
|
|
499
|
+
def to_pandas_sql(self):
|
|
500
|
+
value = self.children[1].value
|
|
501
|
+
column_name = self.children[0].name
|
|
502
|
+
|
|
503
|
+
if FieldType.is_date(self.children[0].type):
|
|
504
|
+
return _compare_with_pandas_dt(self.children[0], '>', value)
|
|
505
|
+
|
|
506
|
+
if isinstance(value, str):
|
|
507
|
+
return f"`{column_name}` > '{value}'"
|
|
508
|
+
|
|
509
|
+
return f"`{column_name}` > {value}"
|
|
510
|
+
|
|
511
|
+
def to_sql(self):
|
|
512
|
+
column_name = self.children[0].name
|
|
513
|
+
value = string_replace(str(self.children[1].value))
|
|
514
|
+
field_type = self.children[0].type
|
|
515
|
+
|
|
516
|
+
if FieldType.is_date(field_type):
|
|
517
|
+
sql = _add_func_to_sql(field_type, column_name, value, ">")
|
|
518
|
+
return sql
|
|
519
|
+
|
|
520
|
+
if isinstance(self.children[1].value, str):
|
|
521
|
+
return f"{column_name} > '{value}'"
|
|
522
|
+
|
|
523
|
+
return f"{column_name} > {value}"
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
class LogicalLessThanOP(_BaseOperatorNode):
|
|
527
|
+
"""< (LESS_THAN)"""
|
|
528
|
+
|
|
529
|
+
def to_pandas_sql(self):
|
|
530
|
+
value = self.children[1].value
|
|
531
|
+
column_name = self.children[0].name
|
|
532
|
+
|
|
533
|
+
if FieldType.is_date(self.children[0].type):
|
|
534
|
+
return _compare_with_pandas_dt(self.children[0], '<', value)
|
|
535
|
+
|
|
536
|
+
if isinstance(value, str):
|
|
537
|
+
return f"`{column_name}` < '{value}'"
|
|
538
|
+
|
|
539
|
+
return f"`{column_name}` < {value}"
|
|
540
|
+
|
|
541
|
+
def to_sql(self):
|
|
542
|
+
column_name = self.children[0].name
|
|
543
|
+
value = string_replace(str(self.children[1].value))
|
|
544
|
+
field_type = self.children[0].type
|
|
545
|
+
|
|
546
|
+
if FieldType.is_date(field_type):
|
|
547
|
+
sql = _add_func_to_sql(field_type, column_name, value, "<")
|
|
548
|
+
return sql
|
|
549
|
+
|
|
550
|
+
if isinstance(self.children[1].value, str):
|
|
551
|
+
return f"{column_name} < '{value}'"
|
|
552
|
+
|
|
553
|
+
return f"{column_name} < {value}"
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
class LogicalGreaterEqualOP(_BaseOperatorNode):
|
|
557
|
+
""">= (GREATER_THAN_OR_EQUAL)"""
|
|
558
|
+
|
|
559
|
+
def to_pandas_sql(self):
|
|
560
|
+
value = self.children[1].value
|
|
561
|
+
column_name = self.children[0].name
|
|
562
|
+
|
|
563
|
+
if FieldType.is_date(self.children[0].type):
|
|
564
|
+
return _compare_with_pandas_dt(self.children[0], '>=', value)
|
|
565
|
+
|
|
566
|
+
if isinstance(value, str):
|
|
567
|
+
return f"`{column_name}` >= '{value}'"
|
|
568
|
+
|
|
569
|
+
return f"`{column_name}` >= {value}"
|
|
570
|
+
|
|
571
|
+
def to_sql(self):
|
|
572
|
+
column_name = self.children[0].name
|
|
573
|
+
value = string_replace(str(self.children[1].value))
|
|
574
|
+
field_type = self.children[0].type
|
|
575
|
+
|
|
576
|
+
if FieldType.is_date(field_type):
|
|
577
|
+
sql = _add_func_to_sql(field_type, column_name, value, ">=")
|
|
578
|
+
return sql
|
|
579
|
+
|
|
580
|
+
if isinstance(self.children[1].value, str):
|
|
581
|
+
return f"{column_name} >= '{value}'"
|
|
582
|
+
|
|
583
|
+
return f"{column_name} >= {value}"
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
class LogicalLessEqualOP(_BaseOperatorNode):
|
|
587
|
+
"""<= (LESS_THAN_OR_EQUAL)"""
|
|
588
|
+
|
|
589
|
+
def to_pandas_sql(self):
|
|
590
|
+
value = self.children[1].value
|
|
591
|
+
column_name = self.children[0].name
|
|
592
|
+
|
|
593
|
+
if FieldType.is_date(self.children[0].type):
|
|
594
|
+
return _compare_with_pandas_dt(self.children[0], '<=', value)
|
|
595
|
+
|
|
596
|
+
if isinstance(value, str):
|
|
597
|
+
return f"`{column_name}` <= '{value}'"
|
|
598
|
+
|
|
599
|
+
return f"`{column_name}` <= {value}"
|
|
600
|
+
|
|
601
|
+
def to_sql(self):
|
|
602
|
+
column_name = self.children[0].name
|
|
603
|
+
value = string_replace(str(self.children[1].value))
|
|
604
|
+
field_type = self.children[0].type
|
|
605
|
+
|
|
606
|
+
if FieldType.is_date(field_type):
|
|
607
|
+
sql = _add_func_to_sql(field_type, column_name, value, "<=")
|
|
608
|
+
return sql
|
|
609
|
+
|
|
610
|
+
if isinstance(self.children[1].value, str):
|
|
611
|
+
return f"{column_name} <= '{value}'"
|
|
612
|
+
|
|
613
|
+
return f"{column_name} <= {value}"
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
class FieldNode(NodeMixin):
|
|
617
|
+
def __init__(self, name, type_=FieldType.unset, precise=None):
|
|
618
|
+
self.name = name
|
|
619
|
+
self.type = maybe_adapt_real_field_type(name, type_)
|
|
620
|
+
self.precise = precise
|
|
621
|
+
|
|
622
|
+
def __str__(self): # pragma: no cover
|
|
623
|
+
return self.name
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
class ComparisonVal(NodeMixin):
|
|
627
|
+
def __init__(self, value):
|
|
628
|
+
self.value = value
|
|
629
|
+
|
|
630
|
+
def __str__(self): # pragma: no cover
|
|
631
|
+
return str(self.value)
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
class LogicalOpFactory:
|
|
635
|
+
"""逻辑操作类的匹配工厂"""
|
|
636
|
+
cls_map = {
|
|
637
|
+
ConnectType.or_: LogicalOrOP,
|
|
638
|
+
ConnectType.and_: LogicalAndOP,
|
|
639
|
+
OperateType.equal: LogicalEqualOP,
|
|
640
|
+
OperateType.not_equal: LogicalNotEqualOP,
|
|
641
|
+
OperateType.in_list: LogicalInListOP,
|
|
642
|
+
OperateType.not_in_list: LogicalNotInListOP,
|
|
643
|
+
OperateType.in_expression: LogicalInExpressionOP,
|
|
644
|
+
OperateType.contains: LogicalContainsOP,
|
|
645
|
+
OperateType.start_with: LogicalStartWithOP,
|
|
646
|
+
OperateType.end_with: LogicalEndWithOP,
|
|
647
|
+
OperateType.is_null: LogicalIsNullOP,
|
|
648
|
+
OperateType.not_null: LogicalNotNullOP,
|
|
649
|
+
OperateType.greater_than: LogicalGreaterThanOP,
|
|
650
|
+
OperateType.less_than: LogicalLessThanOP,
|
|
651
|
+
OperateType.greater_equal: LogicalGreaterEqualOP,
|
|
652
|
+
OperateType.less_equal: LogicalLessEqualOP
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
def __new__(cls, value) -> NodeMixin:
|
|
656
|
+
target_cls = cls.cls_map.get(value)
|
|
657
|
+
if target_cls is None: # pragma: no cover
|
|
658
|
+
raise ValueError(f"Unknown Value for python chart: {value}")
|
|
659
|
+
return target_cls(value)
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
class ASTRoot(NodeMixin):
|
|
663
|
+
pass
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
class FilterParser:
|
|
667
|
+
"""筛选逻辑树解析类"""
|
|
668
|
+
def __init__(
|
|
669
|
+
self,
|
|
670
|
+
source: Union[Dict, FilterItem, SelectItem],
|
|
671
|
+
_all_enable_dim_columns: list = None,
|
|
672
|
+
exclude_dim_expr: bool = False,
|
|
673
|
+
date_round: bool = False
|
|
674
|
+
):
|
|
675
|
+
if isinstance(source, dict):
|
|
676
|
+
source = parse_obj_as(FilterItem, source)
|
|
677
|
+
self.source = source
|
|
678
|
+
self.root = ASTRoot()
|
|
679
|
+
self._all_enable_dim_columns = _all_enable_dim_columns or []
|
|
680
|
+
self.axis_dim_fields_to_expr = collections.defaultdict(list)
|
|
681
|
+
# 解析时是否去除维度表达式(维度类型字段)
|
|
682
|
+
self.exclude_dim_expr = exclude_dim_expr
|
|
683
|
+
self.date_round = date_round
|
|
684
|
+
|
|
685
|
+
def parse(self):
|
|
686
|
+
self.parse_node(self.source).set_parent(self.root)
|
|
687
|
+
|
|
688
|
+
def parse_node(self, source: Union[FilterItem, SelectItem]) -> NodeMixin:
|
|
689
|
+
if source.connectType:
|
|
690
|
+
logic_node_obj = LogicalOpFactory(source.connectType)
|
|
691
|
+
|
|
692
|
+
if isinstance(source, FilterItem):
|
|
693
|
+
for node_source in source.filterGroups:
|
|
694
|
+
node = self.parse_node(node_source)
|
|
695
|
+
if node:
|
|
696
|
+
node.set_parent(logic_node_obj)
|
|
697
|
+
return logic_node_obj
|
|
698
|
+
else:
|
|
699
|
+
for node_source in source.selectGroups:
|
|
700
|
+
node = self.parse_condition(node_source)
|
|
701
|
+
if node:
|
|
702
|
+
node.set_parent(logic_node_obj)
|
|
703
|
+
return logic_node_obj
|
|
704
|
+
else:
|
|
705
|
+
return self.parse_condition(source)
|
|
706
|
+
|
|
707
|
+
def parse_condition(self, source: Union[FilterItem, SelectColumnItem]) -> Optional[NodeMixin]:
|
|
708
|
+
if self.date_round is False:
|
|
709
|
+
if FieldType.is_date(source.formatType) and (precise := dt_precision.get(source.columnName)):
|
|
710
|
+
if precise < TimeType[source.formatType]:
|
|
711
|
+
raise ValueError('筛选条件包含了低精度日期值与高精度日期值的比较操作\n'
|
|
712
|
+
f'当前数据列名: <{source.columnName}>, 配置精度: {precise.name}')
|
|
713
|
+
|
|
714
|
+
conditions = source.conditions if isinstance(source, FilterItem) else source.real_conditions
|
|
715
|
+
# FilterItem中的维度列特殊处理
|
|
716
|
+
if isinstance(source, FilterItem) and source.columnName in self._all_enable_dim_columns:
|
|
717
|
+
# 维度类型字段只有一个条件
|
|
718
|
+
self.axis_dim_fields_to_expr[source.columnName].append(conditions[0].value)
|
|
719
|
+
# 如果该字段是维度类型且本次解析需要去除维度表达式,则无需解析
|
|
720
|
+
if self.exclude_dim_expr:
|
|
721
|
+
return
|
|
722
|
+
|
|
723
|
+
# 如果有多个条件,使用新的子级逻辑连接
|
|
724
|
+
if len(conditions) > 1:
|
|
725
|
+
logical_op = LogicalOpFactory(source.operateType)
|
|
726
|
+
for cond in conditions:
|
|
727
|
+
comp_op = LogicalOpFactory(cond.operate)
|
|
728
|
+
comp_op.set_parent(logical_op)
|
|
729
|
+
FieldNode(source.columnName, source.formatType, dt_precision.get(source.columnName)).set_parent(comp_op)
|
|
730
|
+
if cond.value is not None:
|
|
731
|
+
ComparisonVal(cond.value).set_parent(comp_op)
|
|
732
|
+
# 如果只有一个条件,不创建新的子级,使用父级的逻辑连接
|
|
733
|
+
else:
|
|
734
|
+
logical_op = LogicalOpFactory(conditions[0].operate)
|
|
735
|
+
FieldNode(source.columnName, source.formatType, dt_precision.get(source.columnName)).set_parent(logical_op)
|
|
736
|
+
if conditions[0].value is not None:
|
|
737
|
+
ComparisonVal(conditions[0].value).set_parent(logical_op)
|
|
738
|
+
|
|
739
|
+
return logical_op
|
|
740
|
+
|
|
741
|
+
def debug(self): # pragma: no cover
|
|
742
|
+
TreeRenderer().show(self.root)
|
|
743
|
+
|
|
744
|
+
def make_query(self, dim_mbrs=None, date_fields=None, sql: Enum = Sql.PANDAS_SQL) -> str:
|
|
745
|
+
query = _BaseOperatorNode(self.root.children[0])
|
|
746
|
+
global dim_members, date_type_fields
|
|
747
|
+
if dim_mbrs is not None:
|
|
748
|
+
dim_members = dim_mbrs
|
|
749
|
+
if date_fields is not None:
|
|
750
|
+
date_type_fields = date_fields
|
|
751
|
+
return query.get_where_expression(sql)
|