pydpm_xl 0.1.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- py_dpm/AST/ASTConstructor.py +503 -0
- py_dpm/AST/ASTObjects.py +827 -0
- py_dpm/AST/ASTTemplate.py +101 -0
- py_dpm/AST/ASTVisitor.py +13 -0
- py_dpm/AST/MLGeneration.py +588 -0
- py_dpm/AST/ModuleAnalyzer.py +79 -0
- py_dpm/AST/ModuleDependencies.py +203 -0
- py_dpm/AST/WhereClauseChecker.py +12 -0
- py_dpm/AST/__init__.py +0 -0
- py_dpm/AST/check_operands.py +302 -0
- py_dpm/DataTypes/ScalarTypes.py +324 -0
- py_dpm/DataTypes/TimeClasses.py +370 -0
- py_dpm/DataTypes/TypePromotion.py +195 -0
- py_dpm/DataTypes/__init__.py +0 -0
- py_dpm/Exceptions/__init__.py +0 -0
- py_dpm/Exceptions/exceptions.py +84 -0
- py_dpm/Exceptions/messages.py +114 -0
- py_dpm/OperationScopes/OperationScopeService.py +247 -0
- py_dpm/OperationScopes/__init__.py +0 -0
- py_dpm/Operators/AggregateOperators.py +138 -0
- py_dpm/Operators/BooleanOperators.py +30 -0
- py_dpm/Operators/ClauseOperators.py +159 -0
- py_dpm/Operators/ComparisonOperators.py +69 -0
- py_dpm/Operators/ConditionalOperators.py +362 -0
- py_dpm/Operators/NumericOperators.py +101 -0
- py_dpm/Operators/Operator.py +388 -0
- py_dpm/Operators/StringOperators.py +27 -0
- py_dpm/Operators/TimeOperators.py +53 -0
- py_dpm/Operators/__init__.py +0 -0
- py_dpm/Utils/ValidationsGenerationUtils.py +429 -0
- py_dpm/Utils/__init__.py +0 -0
- py_dpm/Utils/operands_mapping.py +73 -0
- py_dpm/Utils/operator_mapping.py +89 -0
- py_dpm/Utils/tokens.py +172 -0
- py_dpm/Utils/utils.py +2 -0
- py_dpm/ValidationsGeneration/PropertiesConstraintsProcessor.py +190 -0
- py_dpm/ValidationsGeneration/Utils.py +364 -0
- py_dpm/ValidationsGeneration/VariantsProcessor.py +265 -0
- py_dpm/ValidationsGeneration/__init__.py +0 -0
- py_dpm/ValidationsGeneration/auxiliary_functions.py +98 -0
- py_dpm/__init__.py +61 -0
- py_dpm/api/__init__.py +140 -0
- py_dpm/api/ast_generator.py +438 -0
- py_dpm/api/complete_ast.py +241 -0
- py_dpm/api/data_dictionary_validation.py +577 -0
- py_dpm/api/migration.py +77 -0
- py_dpm/api/semantic.py +224 -0
- py_dpm/api/syntax.py +182 -0
- py_dpm/client.py +106 -0
- py_dpm/data_handlers.py +99 -0
- py_dpm/db_utils.py +117 -0
- py_dpm/grammar/__init__.py +0 -0
- py_dpm/grammar/dist/__init__.py +0 -0
- py_dpm/grammar/dist/dpm_xlLexer.interp +428 -0
- py_dpm/grammar/dist/dpm_xlLexer.py +804 -0
- py_dpm/grammar/dist/dpm_xlLexer.tokens +106 -0
- py_dpm/grammar/dist/dpm_xlParser.interp +249 -0
- py_dpm/grammar/dist/dpm_xlParser.py +5224 -0
- py_dpm/grammar/dist/dpm_xlParser.tokens +106 -0
- py_dpm/grammar/dist/dpm_xlParserListener.py +742 -0
- py_dpm/grammar/dist/dpm_xlParserVisitor.py +419 -0
- py_dpm/grammar/dist/listeners.py +10 -0
- py_dpm/grammar/dpm_xlLexer.g4 +435 -0
- py_dpm/grammar/dpm_xlParser.g4 +260 -0
- py_dpm/migration.py +282 -0
- py_dpm/models.py +2139 -0
- py_dpm/semantics/DAG/DAGAnalyzer.py +158 -0
- py_dpm/semantics/DAG/__init__.py +0 -0
- py_dpm/semantics/SemanticAnalyzer.py +320 -0
- py_dpm/semantics/Symbols.py +223 -0
- py_dpm/semantics/__init__.py +0 -0
- py_dpm/utils/__init__.py +0 -0
- py_dpm/utils/ast_serialization.py +481 -0
- py_dpm/views/data_types.sql +12 -0
- py_dpm/views/datapoints.sql +65 -0
- py_dpm/views/hierarchy_operand_reference.sql +11 -0
- py_dpm/views/hierarchy_preconditions.sql +13 -0
- py_dpm/views/hierarchy_variables.sql +26 -0
- py_dpm/views/hierarchy_variables_context.sql +14 -0
- py_dpm/views/key_components.sql +18 -0
- py_dpm/views/module_from_table.sql +11 -0
- py_dpm/views/open_keys.sql +13 -0
- py_dpm/views/operation_info.sql +27 -0
- py_dpm/views/operation_list.sql +18 -0
- py_dpm/views/operations_versions_from_module_version.sql +30 -0
- py_dpm/views/precondition_info.sql +17 -0
- py_dpm/views/report_type_operand_reference_info.sql +18 -0
- py_dpm/views/subcategory_info.sql +17 -0
- py_dpm/views/table_info.sql +19 -0
- pydpm_xl-0.1.10.dist-info/LICENSE +674 -0
- pydpm_xl-0.1.10.dist-info/METADATA +50 -0
- pydpm_xl-0.1.10.dist-info/RECORD +94 -0
- pydpm_xl-0.1.10.dist-info/WHEEL +4 -0
- pydpm_xl-0.1.10.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,588 @@
|
|
|
1
|
+
from sqlalchemy.orm import Session
|
|
2
|
+
|
|
3
|
+
from py_dpm.AST.ASTObjects import *
|
|
4
|
+
from py_dpm.AST.ASTTemplate import ASTTemplate
|
|
5
|
+
from py_dpm.Exceptions.exceptions import SemanticError
|
|
6
|
+
from py_dpm.models import ItemCategory, OperandReference, OperandReferenceLocation, OperationNode, Operator, \
|
|
7
|
+
OperatorArgument, \
|
|
8
|
+
VariableVersion
|
|
9
|
+
from py_dpm.OperationScopes.OperationScopeService import OperationScopeService
|
|
10
|
+
from py_dpm.data_handlers import filter_all_data, generate_xyz
|
|
11
|
+
from py_dpm.db_utils import get_session
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def gather_element(node, attribute):
|
|
15
|
+
if hasattr(node, attribute):
|
|
16
|
+
return getattr(node, attribute)
|
|
17
|
+
return None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def property_ref_period_mangement(name):
|
|
21
|
+
if name == "refPeriod":
|
|
22
|
+
return True
|
|
23
|
+
return False
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class MLGeneration(ASTTemplate):
|
|
27
|
+
"""
|
|
28
|
+
DPM-ML generation is applied after checking the expression is semantically valid.
|
|
29
|
+
|
|
30
|
+
Extracts the data provided by the semantic analysis and the Operation Version object to be associated with this expression.
|
|
31
|
+
Based on the Visit pattern, creates the OperandNode, OperandReference and OperandReferenceLocation for each sub-operation and the
|
|
32
|
+
involved operands.
|
|
33
|
+
|
|
34
|
+
:parameter session: SQLAlchemy Session to be used to connect to the DB.
|
|
35
|
+
:parameter data: Semantic analysis data that includes the all cell references used in the operation.
|
|
36
|
+
:parameter op_version_id: ID of the Operation Version to be associated with each OperandNode.
|
|
37
|
+
:parameter release_id: ID of the Release to be used
|
|
38
|
+
:var df_operators: All operators present in the Operator table
|
|
39
|
+
:var df_arguments: All arguments present in the OperatorArgument table.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def __init__(self, session, data, op_version_id, release_id, operations_data=None, store=False):
|
|
43
|
+
super().__init__()
|
|
44
|
+
self.session: Session = get_session() if store else session
|
|
45
|
+
self.session_queries = get_session()
|
|
46
|
+
self.data = data
|
|
47
|
+
self.table_ids = [int(x) for x in data['table_vid'].unique()] if data is not None else []
|
|
48
|
+
self.op_version_id = op_version_id
|
|
49
|
+
self.df_operators = Operator.get_operators(self.session)
|
|
50
|
+
self.df_arguments = OperatorArgument.get_arguments(self.session)
|
|
51
|
+
self.release_id = release_id
|
|
52
|
+
self.is_scripting = False if op_version_id else True
|
|
53
|
+
|
|
54
|
+
self.operations_data = operations_data
|
|
55
|
+
self.operation_tables = {}
|
|
56
|
+
|
|
57
|
+
self.table_vid_dict = {}
|
|
58
|
+
self.precondition_items = []
|
|
59
|
+
self.existing_scopes = []
|
|
60
|
+
self.new_scopes = []
|
|
61
|
+
self.result = {}
|
|
62
|
+
self.store = store
|
|
63
|
+
|
|
64
|
+
def populate_operation_scope(self):
|
|
65
|
+
operation_scope_service = OperationScopeService(operation_version_id=self.op_version_id, session=self.session)
|
|
66
|
+
self.existing_scopes, self.new_scopes = operation_scope_service.calculate_operation_scope(
|
|
67
|
+
tables_vids=list(self.table_vid_dict.values()), precondition_items=self.precondition_items, only_last_release=False)
|
|
68
|
+
|
|
69
|
+
def extract_operand_data(self, table, rows, cols, sheets):
|
|
70
|
+
data_filtered = filter_all_data(self.data, table, rows, cols, sheets)
|
|
71
|
+
data_filtered = data_filtered[['row_code', 'column_code', 'sheet_code', 'variable_id', 'cell_id']]
|
|
72
|
+
|
|
73
|
+
list_xyz = generate_xyz(data_filtered)
|
|
74
|
+
|
|
75
|
+
return list_xyz
|
|
76
|
+
|
|
77
|
+
def create_operation_node(self, node, is_leaf=False):
|
|
78
|
+
|
|
79
|
+
parent_node = gather_element(node, "parent")
|
|
80
|
+
op = gather_element(node, "op")
|
|
81
|
+
scalar = gather_element(node, "scalar")
|
|
82
|
+
interval = gather_element(node, "interval")
|
|
83
|
+
interval = bool(interval) if interval is not None else False
|
|
84
|
+
fallback_value = gather_element(node, "default")
|
|
85
|
+
|
|
86
|
+
if isinstance(fallback_value, Constant):
|
|
87
|
+
fallback_value = fallback_value.value
|
|
88
|
+
|
|
89
|
+
if fallback_value is not None and isinstance(fallback_value, str):
|
|
90
|
+
if len(fallback_value) == 0:
|
|
91
|
+
fallback_value = '\"\"'
|
|
92
|
+
|
|
93
|
+
operator_id = None
|
|
94
|
+
if not getattr(node, "operator_name", None):
|
|
95
|
+
operator = self.df_operators[self.df_operators['Symbol'] == op]['OperatorID'].values
|
|
96
|
+
else:
|
|
97
|
+
operator = \
|
|
98
|
+
self.df_operators[
|
|
99
|
+
(self.df_operators['Symbol'] == op) & (self.df_operators['Name'] == getattr(node, "operator_name", None))][
|
|
100
|
+
'OperatorID'].values
|
|
101
|
+
|
|
102
|
+
if len(operator) > 0:
|
|
103
|
+
operator_id = int(operator[0])
|
|
104
|
+
argument = getattr(node, "argument", None)
|
|
105
|
+
argument_id = None
|
|
106
|
+
if argument:
|
|
107
|
+
parent_operator_id = parent_node.OperatorID
|
|
108
|
+
argument_info = \
|
|
109
|
+
self.df_arguments[(self.df_arguments['Name'] == argument) & (self.df_arguments['OperatorID'] == parent_operator_id)][
|
|
110
|
+
'ArgumentID'].values
|
|
111
|
+
if len(argument_info) > 0:
|
|
112
|
+
argument_id = int(argument_info[0])
|
|
113
|
+
|
|
114
|
+
operand_node = OperationNode(
|
|
115
|
+
OperatorID=operator_id,
|
|
116
|
+
OperationVID=self.op_version_id,
|
|
117
|
+
parent=parent_node,
|
|
118
|
+
Scalar=scalar,
|
|
119
|
+
UseIntervalArithmetics=interval,
|
|
120
|
+
FallbackValue=fallback_value,
|
|
121
|
+
IsLeaf=is_leaf,
|
|
122
|
+
ArgumentID=argument_id
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
self.session.add(operand_node)
|
|
126
|
+
return operand_node
|
|
127
|
+
|
|
128
|
+
def visit_Start(self, node: Start):
|
|
129
|
+
for child in node.children:
|
|
130
|
+
try:
|
|
131
|
+
if self.is_scripting:
|
|
132
|
+
self._set_op_version_id_from_operation(child)
|
|
133
|
+
self.visit(child)
|
|
134
|
+
if not self.is_scripting:
|
|
135
|
+
if self.store:
|
|
136
|
+
self.session.commit()
|
|
137
|
+
if len(self.precondition_items) == 0:
|
|
138
|
+
self.populate_operation_scope()
|
|
139
|
+
self.store_objects_as_json()
|
|
140
|
+
if self.store:
|
|
141
|
+
for elto in self.new_scopes:
|
|
142
|
+
self.session.add(elto)
|
|
143
|
+
self.session.commit()
|
|
144
|
+
except Exception as e:
|
|
145
|
+
self.session.rollback()
|
|
146
|
+
self.session.close()
|
|
147
|
+
self.session_queries.close()
|
|
148
|
+
raise e
|
|
149
|
+
self.session_queries.close()
|
|
150
|
+
|
|
151
|
+
def visit_PersistentAssignment(self, node: PersistentAssignment):
|
|
152
|
+
|
|
153
|
+
operation_node = self.create_operation_node(node)
|
|
154
|
+
|
|
155
|
+
setattr(node.left, "argument", "left")
|
|
156
|
+
setattr(node.right, "argument", "right")
|
|
157
|
+
|
|
158
|
+
setattr(node.left, "parent", operation_node)
|
|
159
|
+
setattr(node.right, "parent", operation_node)
|
|
160
|
+
|
|
161
|
+
self.visit(node.left)
|
|
162
|
+
self.visit(node.right)
|
|
163
|
+
|
|
164
|
+
def visit_TemporaryAssignment(self, node: TemporaryAssignment):
|
|
165
|
+
self.visit(node.right)
|
|
166
|
+
|
|
167
|
+
def visit_ParExpr(self, node: ParExpr):
|
|
168
|
+
setattr(node, "op", "()")
|
|
169
|
+
operand_node = self.create_operation_node(node)
|
|
170
|
+
setattr(node.expression, "argument", "expression")
|
|
171
|
+
setattr(node.expression, "parent", operand_node)
|
|
172
|
+
self.visit(node.expression)
|
|
173
|
+
|
|
174
|
+
def visit_BinOp(self, node: BinOp):
|
|
175
|
+
if node.op == '+':
|
|
176
|
+
setattr(node, "operator_name", "Addition")
|
|
177
|
+
elif node.op == '-':
|
|
178
|
+
setattr(node, "operator_name", "Subtraction")
|
|
179
|
+
operand_node = self.create_operation_node(node)
|
|
180
|
+
|
|
181
|
+
if node.op == 'in':
|
|
182
|
+
setattr(node.left, "argument", "operand")
|
|
183
|
+
setattr(node.right, "argument", "set")
|
|
184
|
+
elif node.op == 'match':
|
|
185
|
+
setattr(node.left, "argument", "operand")
|
|
186
|
+
setattr(node.right, "argument", "pattern")
|
|
187
|
+
else:
|
|
188
|
+
setattr(node.left, "argument", "left")
|
|
189
|
+
setattr(node.right, "argument", "right")
|
|
190
|
+
setattr(node.left, "parent", operand_node)
|
|
191
|
+
setattr(node.right, "parent", operand_node)
|
|
192
|
+
self.visit(node.left)
|
|
193
|
+
self.visit(node.right)
|
|
194
|
+
|
|
195
|
+
def visit_UnaryOp(self, node: UnaryOp):
|
|
196
|
+
if node.op == '+':
|
|
197
|
+
setattr(node, "operator_name", "Unary plus")
|
|
198
|
+
elif node.op == '-':
|
|
199
|
+
setattr(node, "operator_name", "Unary minus")
|
|
200
|
+
operand_node = self.create_operation_node(node)
|
|
201
|
+
setattr(node.operand, "argument", "operand")
|
|
202
|
+
setattr(node.operand, "parent", operand_node)
|
|
203
|
+
self.visit(node.operand)
|
|
204
|
+
|
|
205
|
+
def visit_CondExpr(self, node: CondExpr):
|
|
206
|
+
setattr(node, "op", "if-then-else")
|
|
207
|
+
operand_if = self.create_operation_node(node)
|
|
208
|
+
setattr(node.condition, "parent", operand_if)
|
|
209
|
+
setattr(node.condition, "argument", "condition")
|
|
210
|
+
self.visit(node.condition)
|
|
211
|
+
setattr(node.then_expr, "argument", "then")
|
|
212
|
+
setattr(node.then_expr, "parent", operand_if)
|
|
213
|
+
self.visit(node.then_expr)
|
|
214
|
+
if node.else_expr:
|
|
215
|
+
setattr(node.else_expr, "argument", "else")
|
|
216
|
+
setattr(node.else_expr, "parent", operand_if)
|
|
217
|
+
self.visit(node.else_expr)
|
|
218
|
+
|
|
219
|
+
def visit_WithExpression(self, node: WithExpression):
|
|
220
|
+
parent = getattr(node, 'parent', None)
|
|
221
|
+
if parent:
|
|
222
|
+
setattr(node.expression, "parent", parent)
|
|
223
|
+
setattr(node.expression, 'argument', getattr(node, 'argument', None))
|
|
224
|
+
self.visit(node.expression)
|
|
225
|
+
|
|
226
|
+
def visit_AggregationOp(self, node: AggregationOp):
|
|
227
|
+
operand_node = self.create_operation_node(node)
|
|
228
|
+
setattr(node.operand, "parent", operand_node)
|
|
229
|
+
setattr(node.operand, "argument", "operand")
|
|
230
|
+
self.visit(node.operand)
|
|
231
|
+
if node.grouping_clause:
|
|
232
|
+
setattr(node.grouping_clause, "op", "group by")
|
|
233
|
+
setattr(node.grouping_clause, "parent", operand_node)
|
|
234
|
+
setattr(node.grouping_clause, "argument", "grouping_clause")
|
|
235
|
+
self.visit(node.grouping_clause)
|
|
236
|
+
|
|
237
|
+
def visit_GroupingClause(self, node: GroupingClause):
|
|
238
|
+
gc_node = self.create_operation_node(node)
|
|
239
|
+
for component in node.components:
|
|
240
|
+
element = AST()
|
|
241
|
+
setattr(element, "parent", gc_node)
|
|
242
|
+
setattr(element, "argument", "component")
|
|
243
|
+
comp_node = self.create_operation_node(element, is_leaf=True)
|
|
244
|
+
# property_id = ItemCategory.get_property_id_from_code(code=node.component, session=self.session)[0]
|
|
245
|
+
if component in ('r', 'c', 's'):
|
|
246
|
+
op_ref = OperandReference(
|
|
247
|
+
op_node=comp_node,
|
|
248
|
+
OperandReference=component
|
|
249
|
+
)
|
|
250
|
+
else:
|
|
251
|
+
property_id = ItemCategory.get_property_id_from_code(code=component, session=self.session_queries)[0]
|
|
252
|
+
op_ref = OperandReference(
|
|
253
|
+
op_node=comp_node,
|
|
254
|
+
OperandReference='property',
|
|
255
|
+
PropertyID=property_id
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
self.session.add(op_ref)
|
|
259
|
+
|
|
260
|
+
def visit_ComplexNumericOp(self, node: ComplexNumericOp):
|
|
261
|
+
operand_node = self.create_operation_node(node)
|
|
262
|
+
for operand in node.operands:
|
|
263
|
+
setattr(operand, "parent", operand_node)
|
|
264
|
+
setattr(operand, "argument", "operand")
|
|
265
|
+
self.visit(operand)
|
|
266
|
+
|
|
267
|
+
def visit_FilterOp(self, node: FilterOp):
|
|
268
|
+
setattr(node, "op", "filter")
|
|
269
|
+
operand_node = self.create_operation_node(node)
|
|
270
|
+
setattr(node.selection, "parent", operand_node)
|
|
271
|
+
setattr(node.selection, "argument", "selection")
|
|
272
|
+
|
|
273
|
+
setattr(node.condition, "parent", operand_node)
|
|
274
|
+
setattr(node.condition, "argument", "condition")
|
|
275
|
+
|
|
276
|
+
self.visit(node.selection)
|
|
277
|
+
self.visit(node.condition)
|
|
278
|
+
|
|
279
|
+
def visit_TimeShiftOp(self, node: TimeShiftOp):
|
|
280
|
+
setattr(node, "op", "time_shift")
|
|
281
|
+
get_node = self.create_operation_node(node)
|
|
282
|
+
|
|
283
|
+
setattr(node.operand, "parent", get_node)
|
|
284
|
+
setattr(node.operand, "argument", "operand")
|
|
285
|
+
self.visit(node.operand)
|
|
286
|
+
|
|
287
|
+
# period indicator
|
|
288
|
+
period_indicator_node = AST()
|
|
289
|
+
setattr(period_indicator_node, "parent", get_node)
|
|
290
|
+
setattr(period_indicator_node, "argument", "period_indicator")
|
|
291
|
+
setattr(period_indicator_node, "scalar", "Q")
|
|
292
|
+
self.create_operation_node(period_indicator_node, is_leaf=True)
|
|
293
|
+
|
|
294
|
+
# shift number
|
|
295
|
+
shift_number_node = AST()
|
|
296
|
+
setattr(shift_number_node, "parent", get_node)
|
|
297
|
+
setattr(shift_number_node, "argument", "shift_number")
|
|
298
|
+
setattr(shift_number_node, "scalar", getattr(node, 'shift_number', None))
|
|
299
|
+
self.create_operation_node(shift_number_node, is_leaf=True)
|
|
300
|
+
|
|
301
|
+
# component
|
|
302
|
+
ast_element = AST()
|
|
303
|
+
setattr(ast_element, "parent", get_node)
|
|
304
|
+
setattr(ast_element, "argument", "dimension")
|
|
305
|
+
setattr(ast_element, "source_reference", "property")
|
|
306
|
+
operand_node = self.create_operation_node(ast_element, is_leaf=True) # TODO: Adapt to refPeriod
|
|
307
|
+
if property_ref_period_mangement(node.component):
|
|
308
|
+
op_ref = OperandReference(
|
|
309
|
+
op_node=operand_node,
|
|
310
|
+
OperandReference="refPeriod"
|
|
311
|
+
)
|
|
312
|
+
else:
|
|
313
|
+
property_id = ItemCategory.get_property_id_from_code(code=node.component, session=self.session_queries)[0]
|
|
314
|
+
op_ref = OperandReference(
|
|
315
|
+
op_node=operand_node,
|
|
316
|
+
OperandReference=getattr(ast_element, "source_reference"),
|
|
317
|
+
PropertyID=property_id
|
|
318
|
+
)
|
|
319
|
+
self.session.add(op_ref)
|
|
320
|
+
|
|
321
|
+
def visit_WhereClauseOp(self, node: WhereClauseOp):
|
|
322
|
+
setattr(node, "op", "where")
|
|
323
|
+
node_op = self.create_operation_node(node)
|
|
324
|
+
setattr(node.operand, "parent", node_op)
|
|
325
|
+
setattr(node.operand, "argument", "operand")
|
|
326
|
+
|
|
327
|
+
setattr(node.condition, "parent", node_op)
|
|
328
|
+
setattr(node.condition, "argument", "condition")
|
|
329
|
+
self.visit(node.operand)
|
|
330
|
+
self.visit(node.condition)
|
|
331
|
+
|
|
332
|
+
def visit_GetOp(self, node: GetOp):
|
|
333
|
+
setattr(node, "op", "get")
|
|
334
|
+
node_op = self.create_operation_node(node)
|
|
335
|
+
setattr(node.operand, "parent", node_op)
|
|
336
|
+
setattr(node.operand, "argument", "operand")
|
|
337
|
+
self.visit(node.operand)
|
|
338
|
+
|
|
339
|
+
element = AST()
|
|
340
|
+
setattr(element, "parent", node_op)
|
|
341
|
+
setattr(element, "argument", "component")
|
|
342
|
+
setattr(element, "source_reference", "property")
|
|
343
|
+
component_node = self.create_operation_node(element, is_leaf=True)
|
|
344
|
+
if property_ref_period_mangement(node.component):
|
|
345
|
+
op_ref = OperandReference(
|
|
346
|
+
op_node=component_node,
|
|
347
|
+
OperandReference="refPeriod"
|
|
348
|
+
)
|
|
349
|
+
else:
|
|
350
|
+
property_id = ItemCategory.get_property_id_from_code(code=node.component, session=self.session_queries)[0]
|
|
351
|
+
op_ref = OperandReference(
|
|
352
|
+
op_node=component_node,
|
|
353
|
+
OperandReference=getattr(element, "source_reference"),
|
|
354
|
+
PropertyID=property_id
|
|
355
|
+
)
|
|
356
|
+
self.session.add(op_ref)
|
|
357
|
+
|
|
358
|
+
def visit_RenameOp(self, node: RenameOp):
|
|
359
|
+
setattr(node, "op", "rename")
|
|
360
|
+
operand_node = self.create_operation_node(node)
|
|
361
|
+
setattr(node.operand, "parent", operand_node)
|
|
362
|
+
setattr(node.operand, "argument", "operand")
|
|
363
|
+
self.visit(node.operand)
|
|
364
|
+
|
|
365
|
+
for rename_node in node.rename_nodes:
|
|
366
|
+
setattr(rename_node, "parent", operand_node)
|
|
367
|
+
setattr(rename_node, "argument", "node")
|
|
368
|
+
self.visit(rename_node)
|
|
369
|
+
|
|
370
|
+
def visit_RenameNode(self, node: RenameNode):
|
|
371
|
+
setattr(node, "op", "node")
|
|
372
|
+
rename_node = self.create_operation_node(node)
|
|
373
|
+
old_name = AST()
|
|
374
|
+
setattr(old_name, "parent", rename_node)
|
|
375
|
+
setattr(old_name, "argument", "old_name")
|
|
376
|
+
setattr(old_name, "source_reference", "property")
|
|
377
|
+
setattr(old_name, "scalar", node.old_name)
|
|
378
|
+
old_property_id = ItemCategory.get_property_id_from_code(code=node.old_name, session=self.session_queries)[0]
|
|
379
|
+
old_name_node = self.create_operation_node(old_name, is_leaf=True)
|
|
380
|
+
|
|
381
|
+
old_operand_ref = OperandReference(
|
|
382
|
+
op_node=old_name_node,
|
|
383
|
+
OperandReference=getattr(old_name, "source_reference"),
|
|
384
|
+
PropertyID=old_property_id
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
self.session.add(old_operand_ref)
|
|
388
|
+
|
|
389
|
+
new_name = AST()
|
|
390
|
+
setattr(new_name, "parent", rename_node)
|
|
391
|
+
setattr(new_name, "argument", "new_name")
|
|
392
|
+
setattr(new_name, "source_reference", "property")
|
|
393
|
+
setattr(new_name, "scalar", node.new_name)
|
|
394
|
+
new_name_node = self.create_operation_node(new_name, is_leaf=True)
|
|
395
|
+
|
|
396
|
+
new_operand_ref = OperandReference(
|
|
397
|
+
op_node=new_name_node,
|
|
398
|
+
OperandReference=getattr(new_name, "source_reference"),
|
|
399
|
+
PropertyID=old_property_id
|
|
400
|
+
)
|
|
401
|
+
self.session.add(new_operand_ref)
|
|
402
|
+
|
|
403
|
+
def visit_PreconditionItem(self, node: PreconditionItem):
|
|
404
|
+
operand_node = self.create_operation_node(node, is_leaf=True)
|
|
405
|
+
operand_reference = "PreconditionItem" # "$_{}".format(node.value)
|
|
406
|
+
precondition_var = VariableVersion.check_precondition(self.session, node.variable_code, self.release_id)
|
|
407
|
+
variable_id = None
|
|
408
|
+
precondition_code = None
|
|
409
|
+
if precondition_var:
|
|
410
|
+
variable_id = precondition_var.VariableID
|
|
411
|
+
precondition_code = precondition_var.Code
|
|
412
|
+
else:
|
|
413
|
+
preconditions_vars = VariableVersion.get_all_preconditions(self.session, self.release_id)
|
|
414
|
+
precondition_found = False
|
|
415
|
+
for precondition in preconditions_vars:
|
|
416
|
+
if precondition.Code in node.variable_code:
|
|
417
|
+
variable_id = precondition.VariableID
|
|
418
|
+
precondition_found = True
|
|
419
|
+
precondition_code = precondition.Code
|
|
420
|
+
break
|
|
421
|
+
|
|
422
|
+
if not precondition_found:
|
|
423
|
+
raise SemanticError("1-3", variable=node.variable_code)
|
|
424
|
+
|
|
425
|
+
operand_ref = OperandReference(
|
|
426
|
+
op_node=operand_node,
|
|
427
|
+
OperandReference=operand_reference,
|
|
428
|
+
VariableID=variable_id
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
self.session.add(operand_ref)
|
|
432
|
+
self.precondition_items.append(precondition_code)
|
|
433
|
+
|
|
434
|
+
def visit_VarRef(self, node: VarRef):
|
|
435
|
+
setattr(node, "source_reference", "variable")
|
|
436
|
+
op_node = self.create_operation_node(node, is_leaf=True)
|
|
437
|
+
node_value = getattr(node, "value", getattr(node, "variable", None))
|
|
438
|
+
variable_id = VariableVersion.get_VariableID(self.session, node_value, self.release_id)
|
|
439
|
+
if variable_id:
|
|
440
|
+
variable_id = variable_id[0]
|
|
441
|
+
else:
|
|
442
|
+
raise SemanticError("1-3", variable=node_value)
|
|
443
|
+
operand_ref = OperandReference(
|
|
444
|
+
op_node=op_node,
|
|
445
|
+
OperandReference=getattr(node, "source_reference"),
|
|
446
|
+
VariableID=variable_id
|
|
447
|
+
)
|
|
448
|
+
|
|
449
|
+
self.session.add(operand_ref)
|
|
450
|
+
|
|
451
|
+
def visit_VarID(self, node: VarID):
|
|
452
|
+
setattr(node, "source_reference", "variable")
|
|
453
|
+
|
|
454
|
+
op_node = self.create_operation_node(node, is_leaf=True)
|
|
455
|
+
|
|
456
|
+
data_xyz = self.extract_operand_data(node.table, node.rows, node.cols, node.sheets)
|
|
457
|
+
|
|
458
|
+
# Extracting data
|
|
459
|
+
significant_rows = node.rows is not None and len(node.rows) >= 1
|
|
460
|
+
significant_cols = node.cols is not None and len(node.cols) >= 1
|
|
461
|
+
significant_sheets = node.sheets is not None and len(node.sheets) >= 1
|
|
462
|
+
|
|
463
|
+
for e in data_xyz:
|
|
464
|
+
operand_ref = OperandReference(
|
|
465
|
+
op_node=op_node,
|
|
466
|
+
x=e['x'] if significant_rows else None,
|
|
467
|
+
y=e['y'] if significant_cols else None,
|
|
468
|
+
z=e['z'] if significant_sheets else None,
|
|
469
|
+
OperandReference=getattr(node, "source_reference"),
|
|
470
|
+
VariableID=e['variable_id']
|
|
471
|
+
)
|
|
472
|
+
|
|
473
|
+
self.session.add(operand_ref)
|
|
474
|
+
|
|
475
|
+
operand_ref_loc = OperandReferenceLocation(
|
|
476
|
+
op_reference=operand_ref,
|
|
477
|
+
CellID=e['cell_id'],
|
|
478
|
+
Table=node.table,
|
|
479
|
+
Row=e['row_code'],
|
|
480
|
+
column=e['column_code'],
|
|
481
|
+
Sheet=e['sheet_code']
|
|
482
|
+
)
|
|
483
|
+
|
|
484
|
+
self.session.add(operand_ref_loc)
|
|
485
|
+
|
|
486
|
+
if node.table not in self.table_vid_dict:
|
|
487
|
+
table_vid = int(self.data[self.data['table_code'] == node.table]['table_vid'].unique()[0])
|
|
488
|
+
self.table_vid_dict[node.table] = table_vid
|
|
489
|
+
|
|
490
|
+
if self.is_scripting:
|
|
491
|
+
self._add_table_vid_to_operation_tables(node.table)
|
|
492
|
+
|
|
493
|
+
def visit_Constant(self, node: Constant):
|
|
494
|
+
setattr(node, "scalar", node.value)
|
|
495
|
+
self.create_operation_node(node, is_leaf=True)
|
|
496
|
+
|
|
497
|
+
def visit_Dimension(self, node: Dimension):
|
|
498
|
+
setattr(node, "source_reference", "property")
|
|
499
|
+
op_node = self.create_operation_node(node, is_leaf=True)
|
|
500
|
+
property = ItemCategory.get_property_from_code(code=node.dimension_code, session=self.session_queries)
|
|
501
|
+
operand_ref = OperandReference(
|
|
502
|
+
op_node=op_node,
|
|
503
|
+
OperandReference=getattr(node, "source_reference"),
|
|
504
|
+
PropertyID=property.ItemID
|
|
505
|
+
)
|
|
506
|
+
self.session.add(operand_ref)
|
|
507
|
+
|
|
508
|
+
def visit_Set(self, node: Set):
|
|
509
|
+
setattr(node, "source_reference", "item")
|
|
510
|
+
setattr(node, "operand_type", "set")
|
|
511
|
+
op_node = self.create_operation_node(node, is_leaf=True)
|
|
512
|
+
|
|
513
|
+
for child in node.children:
|
|
514
|
+
item_id = ItemCategory.get_item_category_id_from_signature(
|
|
515
|
+
signature=child.item, session=self.session_queries
|
|
516
|
+
)[0]
|
|
517
|
+
operand_ref = OperandReference(
|
|
518
|
+
op_node=op_node,
|
|
519
|
+
OperandReference=getattr(node, "source_reference"),
|
|
520
|
+
ItemID=item_id
|
|
521
|
+
)
|
|
522
|
+
self.session.add(operand_ref)
|
|
523
|
+
|
|
524
|
+
def visit_Scalar(self, node: Scalar):
|
|
525
|
+
setattr(node, "source_reference", "item")
|
|
526
|
+
op_node = self.create_operation_node(node, is_leaf=True)
|
|
527
|
+
item_id = ItemCategory.get_item_category_id_from_signature(
|
|
528
|
+
signature=node.item, session=self.session_queries
|
|
529
|
+
)[0]
|
|
530
|
+
operand_ref = OperandReference(
|
|
531
|
+
op_node=op_node,
|
|
532
|
+
OperandReference=getattr(node, "source_reference"),
|
|
533
|
+
ItemID=item_id
|
|
534
|
+
)
|
|
535
|
+
self.session.add(operand_ref)
|
|
536
|
+
|
|
537
|
+
def visit_OperationRef(self, node: OperationRef):
|
|
538
|
+
setattr(node, "source_reference", "operation")
|
|
539
|
+
op_node = self.create_operation_node(node, is_leaf=True)
|
|
540
|
+
|
|
541
|
+
op_version_id = self._get_op_version_id(node.operation_code)
|
|
542
|
+
|
|
543
|
+
operand_ref = OperandReference(
|
|
544
|
+
op_node=op_node,
|
|
545
|
+
OperandReference=op_version_id
|
|
546
|
+
)
|
|
547
|
+
self.session.add(operand_ref)
|
|
548
|
+
|
|
549
|
+
def _get_op_version_id(self, operation_code):
|
|
550
|
+
op_version_id = self.operations_data[self.operations_data['Code'] == operation_code]['OperationVID'].values[0]
|
|
551
|
+
op_version_id = int(op_version_id)
|
|
552
|
+
return op_version_id
|
|
553
|
+
|
|
554
|
+
def _set_op_version_id_from_operation(self, child):
|
|
555
|
+
|
|
556
|
+
operation_code = child.left.value
|
|
557
|
+
op_version_id = self._get_op_version_id(operation_code)
|
|
558
|
+
self.op_version_id = op_version_id
|
|
559
|
+
|
|
560
|
+
def _add_table_vid_to_operation_tables(self, table_code):
|
|
561
|
+
if self.op_version_id not in self.operation_tables:
|
|
562
|
+
self.operation_tables[self.op_version_id] = []
|
|
563
|
+
table_vid = int(self.data[self.data['table_code'] == table_code]['table_vid'].unique()[0])
|
|
564
|
+
if table_vid not in self.operation_tables[self.op_version_id]:
|
|
565
|
+
self.operation_tables[self.op_version_id].append(table_vid)
|
|
566
|
+
|
|
567
|
+
def store_objects_as_json(self):
|
|
568
|
+
operation_nodes = [o for o in self.session.new if isinstance(o, OperationNode) and not o.parent]
|
|
569
|
+
self.result['operation_nodes'] = operation_nodes
|
|
570
|
+
self.result['operation_scopes'] = {}
|
|
571
|
+
self.result['operation_scopes']['new'] = self.new_scopes
|
|
572
|
+
self.result['operation_scopes']['existing'] = self.existing_scopes
|
|
573
|
+
self.session.expunge_all()
|
|
574
|
+
|
|
575
|
+
def compare_ast(self, reference: OperationNode):
|
|
576
|
+
"""
|
|
577
|
+
Compares the ML generated by the AST of the expression provided with the ML generated by the AST generated with the ML stored in the db.
|
|
578
|
+
:return: True if the ASTs are equal, False otherwise.
|
|
579
|
+
"""
|
|
580
|
+
op_nodes = self.result['operation_nodes']
|
|
581
|
+
|
|
582
|
+
if len(op_nodes) == 0:
|
|
583
|
+
raise Exception("No AST Generated")
|
|
584
|
+
|
|
585
|
+
if op_nodes[0] != reference:
|
|
586
|
+
return False #is_same_ast = False
|
|
587
|
+
|
|
588
|
+
return True #is_same_ast = True
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
from py_dpm.AST.ASTObjects import Start, VarID, WithExpression
|
|
2
|
+
from py_dpm.AST.ASTTemplate import ASTTemplate
|
|
3
|
+
from py_dpm.models import ViewModules
|
|
4
|
+
from py_dpm.Utils.operands_mapping import LabelHandler
|
|
5
|
+
from py_dpm.Utils.tokens import CROSS_MODULE, INTRA_MODULE, REPEATED_INTRA_MODULE
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ModuleAnalyzer(ASTTemplate):
|
|
9
|
+
def __init__(self, session):
|
|
10
|
+
|
|
11
|
+
super(ASTTemplate).__init__()
|
|
12
|
+
self.modules = []
|
|
13
|
+
self.session = session
|
|
14
|
+
self.mode = None
|
|
15
|
+
self.module_info = {}
|
|
16
|
+
LabelHandler().reset_instance()
|
|
17
|
+
|
|
18
|
+
def new_label(self):
|
|
19
|
+
return LabelHandler().labels.__next__()
|
|
20
|
+
|
|
21
|
+
def extract_modules(self, tables):
|
|
22
|
+
return ViewModules().get_modules(self.session, tables)
|
|
23
|
+
|
|
24
|
+
def module_analysis(self):
|
|
25
|
+
unique_modules = []
|
|
26
|
+
|
|
27
|
+
for operand_info in self.module_info.values():
|
|
28
|
+
if operand_info == 'Module not found':
|
|
29
|
+
print(f"Module not found: {self.module_info}")
|
|
30
|
+
return
|
|
31
|
+
unique_modules += operand_info
|
|
32
|
+
unique_modules = list(set(unique_modules))
|
|
33
|
+
if len(unique_modules) == 1:
|
|
34
|
+
self.mode = INTRA_MODULE
|
|
35
|
+
self.modules = unique_modules
|
|
36
|
+
self.find_common_modules(unique_modules)
|
|
37
|
+
|
|
38
|
+
def visit_Start(self, node: Start):
|
|
39
|
+
self.visit(node.children[0])
|
|
40
|
+
if not isinstance(node.children[0], WithExpression):
|
|
41
|
+
self.module_analysis()
|
|
42
|
+
return self.mode, self.modules
|
|
43
|
+
|
|
44
|
+
def visit_WithExpression(self, node: WithExpression):
|
|
45
|
+
if node.partial_selection.table is not None:
|
|
46
|
+
modules = self.extract_modules([node.partial_selection.table])
|
|
47
|
+
self.modules = modules
|
|
48
|
+
if len(modules) > 1:
|
|
49
|
+
self.mode = REPEATED_INTRA_MODULE
|
|
50
|
+
elif len(modules) == 1:
|
|
51
|
+
self.mode = INTRA_MODULE
|
|
52
|
+
return
|
|
53
|
+
self.visit(node.expression)
|
|
54
|
+
self.module_analysis()
|
|
55
|
+
|
|
56
|
+
def visit_VarID(self, node: VarID):
|
|
57
|
+
modules = self.extract_modules([node.table])
|
|
58
|
+
if len(modules) > 0:
|
|
59
|
+
self.module_info[self.new_label()] = modules
|
|
60
|
+
else:
|
|
61
|
+
self.module_info[self.new_label()] = 'Module not found'
|
|
62
|
+
|
|
63
|
+
def find_common_modules(self, unique_modules):
|
|
64
|
+
common_modules = []
|
|
65
|
+
for operand_info in self.module_info.values():
|
|
66
|
+
if len(common_modules) == 0:
|
|
67
|
+
common_modules = operand_info
|
|
68
|
+
continue
|
|
69
|
+
common_modules = list(set(common_modules) & set(operand_info))
|
|
70
|
+
if len(common_modules) == 0:
|
|
71
|
+
if len(unique_modules) > 1:
|
|
72
|
+
self.mode = CROSS_MODULE
|
|
73
|
+
self.modules = unique_modules
|
|
74
|
+
return
|
|
75
|
+
elif len(common_modules) == 1:
|
|
76
|
+
self.mode = INTRA_MODULE
|
|
77
|
+
else:
|
|
78
|
+
self.mode = CROSS_MODULE
|
|
79
|
+
self.modules = common_modules
|