pydpm_xl 0.1.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. py_dpm/AST/ASTConstructor.py +503 -0
  2. py_dpm/AST/ASTObjects.py +827 -0
  3. py_dpm/AST/ASTTemplate.py +101 -0
  4. py_dpm/AST/ASTVisitor.py +13 -0
  5. py_dpm/AST/MLGeneration.py +588 -0
  6. py_dpm/AST/ModuleAnalyzer.py +79 -0
  7. py_dpm/AST/ModuleDependencies.py +203 -0
  8. py_dpm/AST/WhereClauseChecker.py +12 -0
  9. py_dpm/AST/__init__.py +0 -0
  10. py_dpm/AST/check_operands.py +302 -0
  11. py_dpm/DataTypes/ScalarTypes.py +324 -0
  12. py_dpm/DataTypes/TimeClasses.py +370 -0
  13. py_dpm/DataTypes/TypePromotion.py +195 -0
  14. py_dpm/DataTypes/__init__.py +0 -0
  15. py_dpm/Exceptions/__init__.py +0 -0
  16. py_dpm/Exceptions/exceptions.py +84 -0
  17. py_dpm/Exceptions/messages.py +114 -0
  18. py_dpm/OperationScopes/OperationScopeService.py +247 -0
  19. py_dpm/OperationScopes/__init__.py +0 -0
  20. py_dpm/Operators/AggregateOperators.py +138 -0
  21. py_dpm/Operators/BooleanOperators.py +30 -0
  22. py_dpm/Operators/ClauseOperators.py +159 -0
  23. py_dpm/Operators/ComparisonOperators.py +69 -0
  24. py_dpm/Operators/ConditionalOperators.py +362 -0
  25. py_dpm/Operators/NumericOperators.py +101 -0
  26. py_dpm/Operators/Operator.py +388 -0
  27. py_dpm/Operators/StringOperators.py +27 -0
  28. py_dpm/Operators/TimeOperators.py +53 -0
  29. py_dpm/Operators/__init__.py +0 -0
  30. py_dpm/Utils/ValidationsGenerationUtils.py +429 -0
  31. py_dpm/Utils/__init__.py +0 -0
  32. py_dpm/Utils/operands_mapping.py +73 -0
  33. py_dpm/Utils/operator_mapping.py +89 -0
  34. py_dpm/Utils/tokens.py +172 -0
  35. py_dpm/Utils/utils.py +2 -0
  36. py_dpm/ValidationsGeneration/PropertiesConstraintsProcessor.py +190 -0
  37. py_dpm/ValidationsGeneration/Utils.py +364 -0
  38. py_dpm/ValidationsGeneration/VariantsProcessor.py +265 -0
  39. py_dpm/ValidationsGeneration/__init__.py +0 -0
  40. py_dpm/ValidationsGeneration/auxiliary_functions.py +98 -0
  41. py_dpm/__init__.py +61 -0
  42. py_dpm/api/__init__.py +140 -0
  43. py_dpm/api/ast_generator.py +438 -0
  44. py_dpm/api/complete_ast.py +241 -0
  45. py_dpm/api/data_dictionary_validation.py +577 -0
  46. py_dpm/api/migration.py +77 -0
  47. py_dpm/api/semantic.py +224 -0
  48. py_dpm/api/syntax.py +182 -0
  49. py_dpm/client.py +106 -0
  50. py_dpm/data_handlers.py +99 -0
  51. py_dpm/db_utils.py +117 -0
  52. py_dpm/grammar/__init__.py +0 -0
  53. py_dpm/grammar/dist/__init__.py +0 -0
  54. py_dpm/grammar/dist/dpm_xlLexer.interp +428 -0
  55. py_dpm/grammar/dist/dpm_xlLexer.py +804 -0
  56. py_dpm/grammar/dist/dpm_xlLexer.tokens +106 -0
  57. py_dpm/grammar/dist/dpm_xlParser.interp +249 -0
  58. py_dpm/grammar/dist/dpm_xlParser.py +5224 -0
  59. py_dpm/grammar/dist/dpm_xlParser.tokens +106 -0
  60. py_dpm/grammar/dist/dpm_xlParserListener.py +742 -0
  61. py_dpm/grammar/dist/dpm_xlParserVisitor.py +419 -0
  62. py_dpm/grammar/dist/listeners.py +10 -0
  63. py_dpm/grammar/dpm_xlLexer.g4 +435 -0
  64. py_dpm/grammar/dpm_xlParser.g4 +260 -0
  65. py_dpm/migration.py +282 -0
  66. py_dpm/models.py +2139 -0
  67. py_dpm/semantics/DAG/DAGAnalyzer.py +158 -0
  68. py_dpm/semantics/DAG/__init__.py +0 -0
  69. py_dpm/semantics/SemanticAnalyzer.py +320 -0
  70. py_dpm/semantics/Symbols.py +223 -0
  71. py_dpm/semantics/__init__.py +0 -0
  72. py_dpm/utils/__init__.py +0 -0
  73. py_dpm/utils/ast_serialization.py +481 -0
  74. py_dpm/views/data_types.sql +12 -0
  75. py_dpm/views/datapoints.sql +65 -0
  76. py_dpm/views/hierarchy_operand_reference.sql +11 -0
  77. py_dpm/views/hierarchy_preconditions.sql +13 -0
  78. py_dpm/views/hierarchy_variables.sql +26 -0
  79. py_dpm/views/hierarchy_variables_context.sql +14 -0
  80. py_dpm/views/key_components.sql +18 -0
  81. py_dpm/views/module_from_table.sql +11 -0
  82. py_dpm/views/open_keys.sql +13 -0
  83. py_dpm/views/operation_info.sql +27 -0
  84. py_dpm/views/operation_list.sql +18 -0
  85. py_dpm/views/operations_versions_from_module_version.sql +30 -0
  86. py_dpm/views/precondition_info.sql +17 -0
  87. py_dpm/views/report_type_operand_reference_info.sql +18 -0
  88. py_dpm/views/subcategory_info.sql +17 -0
  89. py_dpm/views/table_info.sql +19 -0
  90. pydpm_xl-0.1.10.dist-info/LICENSE +674 -0
  91. pydpm_xl-0.1.10.dist-info/METADATA +50 -0
  92. pydpm_xl-0.1.10.dist-info/RECORD +94 -0
  93. pydpm_xl-0.1.10.dist-info/WHEEL +4 -0
  94. pydpm_xl-0.1.10.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,588 @@
1
+ from sqlalchemy.orm import Session
2
+
3
+ from py_dpm.AST.ASTObjects import *
4
+ from py_dpm.AST.ASTTemplate import ASTTemplate
5
+ from py_dpm.Exceptions.exceptions import SemanticError
6
+ from py_dpm.models import ItemCategory, OperandReference, OperandReferenceLocation, OperationNode, Operator, \
7
+ OperatorArgument, \
8
+ VariableVersion
9
+ from py_dpm.OperationScopes.OperationScopeService import OperationScopeService
10
+ from py_dpm.data_handlers import filter_all_data, generate_xyz
11
+ from py_dpm.db_utils import get_session
12
+
13
+
14
+ def gather_element(node, attribute):
15
+ if hasattr(node, attribute):
16
+ return getattr(node, attribute)
17
+ return None
18
+
19
+
20
+ def property_ref_period_mangement(name):
21
+ if name == "refPeriod":
22
+ return True
23
+ return False
24
+
25
+
26
+ class MLGeneration(ASTTemplate):
27
+ """
28
+ DPM-ML generation is applied after checking the expression is semantically valid.
29
+
30
+ Extracts the data provided by the semantic analysis and the Operation Version object to be associated with this expression.
31
+ Based on the Visit pattern, creates the OperandNode, OperandReference and OperandReferenceLocation for each sub-operation and the
32
+ involved operands.
33
+
34
+ :parameter session: SQLAlchemy Session to be used to connect to the DB.
35
+ :parameter data: Semantic analysis data that includes the all cell references used in the operation.
36
+ :parameter op_version_id: ID of the Operation Version to be associated with each OperandNode.
37
+ :parameter release_id: ID of the Release to be used
38
+ :var df_operators: All operators present in the Operator table
39
+ :var df_arguments: All arguments present in the OperatorArgument table.
40
+ """
41
+
42
+ def __init__(self, session, data, op_version_id, release_id, operations_data=None, store=False):
43
+ super().__init__()
44
+ self.session: Session = get_session() if store else session
45
+ self.session_queries = get_session()
46
+ self.data = data
47
+ self.table_ids = [int(x) for x in data['table_vid'].unique()] if data is not None else []
48
+ self.op_version_id = op_version_id
49
+ self.df_operators = Operator.get_operators(self.session)
50
+ self.df_arguments = OperatorArgument.get_arguments(self.session)
51
+ self.release_id = release_id
52
+ self.is_scripting = False if op_version_id else True
53
+
54
+ self.operations_data = operations_data
55
+ self.operation_tables = {}
56
+
57
+ self.table_vid_dict = {}
58
+ self.precondition_items = []
59
+ self.existing_scopes = []
60
+ self.new_scopes = []
61
+ self.result = {}
62
+ self.store = store
63
+
64
+ def populate_operation_scope(self):
65
+ operation_scope_service = OperationScopeService(operation_version_id=self.op_version_id, session=self.session)
66
+ self.existing_scopes, self.new_scopes = operation_scope_service.calculate_operation_scope(
67
+ tables_vids=list(self.table_vid_dict.values()), precondition_items=self.precondition_items, only_last_release=False)
68
+
69
+ def extract_operand_data(self, table, rows, cols, sheets):
70
+ data_filtered = filter_all_data(self.data, table, rows, cols, sheets)
71
+ data_filtered = data_filtered[['row_code', 'column_code', 'sheet_code', 'variable_id', 'cell_id']]
72
+
73
+ list_xyz = generate_xyz(data_filtered)
74
+
75
+ return list_xyz
76
+
77
+ def create_operation_node(self, node, is_leaf=False):
78
+
79
+ parent_node = gather_element(node, "parent")
80
+ op = gather_element(node, "op")
81
+ scalar = gather_element(node, "scalar")
82
+ interval = gather_element(node, "interval")
83
+ interval = bool(interval) if interval is not None else False
84
+ fallback_value = gather_element(node, "default")
85
+
86
+ if isinstance(fallback_value, Constant):
87
+ fallback_value = fallback_value.value
88
+
89
+ if fallback_value is not None and isinstance(fallback_value, str):
90
+ if len(fallback_value) == 0:
91
+ fallback_value = '\"\"'
92
+
93
+ operator_id = None
94
+ if not getattr(node, "operator_name", None):
95
+ operator = self.df_operators[self.df_operators['Symbol'] == op]['OperatorID'].values
96
+ else:
97
+ operator = \
98
+ self.df_operators[
99
+ (self.df_operators['Symbol'] == op) & (self.df_operators['Name'] == getattr(node, "operator_name", None))][
100
+ 'OperatorID'].values
101
+
102
+ if len(operator) > 0:
103
+ operator_id = int(operator[0])
104
+ argument = getattr(node, "argument", None)
105
+ argument_id = None
106
+ if argument:
107
+ parent_operator_id = parent_node.OperatorID
108
+ argument_info = \
109
+ self.df_arguments[(self.df_arguments['Name'] == argument) & (self.df_arguments['OperatorID'] == parent_operator_id)][
110
+ 'ArgumentID'].values
111
+ if len(argument_info) > 0:
112
+ argument_id = int(argument_info[0])
113
+
114
+ operand_node = OperationNode(
115
+ OperatorID=operator_id,
116
+ OperationVID=self.op_version_id,
117
+ parent=parent_node,
118
+ Scalar=scalar,
119
+ UseIntervalArithmetics=interval,
120
+ FallbackValue=fallback_value,
121
+ IsLeaf=is_leaf,
122
+ ArgumentID=argument_id
123
+ )
124
+
125
+ self.session.add(operand_node)
126
+ return operand_node
127
+
128
+ def visit_Start(self, node: Start):
129
+ for child in node.children:
130
+ try:
131
+ if self.is_scripting:
132
+ self._set_op_version_id_from_operation(child)
133
+ self.visit(child)
134
+ if not self.is_scripting:
135
+ if self.store:
136
+ self.session.commit()
137
+ if len(self.precondition_items) == 0:
138
+ self.populate_operation_scope()
139
+ self.store_objects_as_json()
140
+ if self.store:
141
+ for elto in self.new_scopes:
142
+ self.session.add(elto)
143
+ self.session.commit()
144
+ except Exception as e:
145
+ self.session.rollback()
146
+ self.session.close()
147
+ self.session_queries.close()
148
+ raise e
149
+ self.session_queries.close()
150
+
151
+ def visit_PersistentAssignment(self, node: PersistentAssignment):
152
+
153
+ operation_node = self.create_operation_node(node)
154
+
155
+ setattr(node.left, "argument", "left")
156
+ setattr(node.right, "argument", "right")
157
+
158
+ setattr(node.left, "parent", operation_node)
159
+ setattr(node.right, "parent", operation_node)
160
+
161
+ self.visit(node.left)
162
+ self.visit(node.right)
163
+
164
+ def visit_TemporaryAssignment(self, node: TemporaryAssignment):
165
+ self.visit(node.right)
166
+
167
+ def visit_ParExpr(self, node: ParExpr):
168
+ setattr(node, "op", "()")
169
+ operand_node = self.create_operation_node(node)
170
+ setattr(node.expression, "argument", "expression")
171
+ setattr(node.expression, "parent", operand_node)
172
+ self.visit(node.expression)
173
+
174
+ def visit_BinOp(self, node: BinOp):
175
+ if node.op == '+':
176
+ setattr(node, "operator_name", "Addition")
177
+ elif node.op == '-':
178
+ setattr(node, "operator_name", "Subtraction")
179
+ operand_node = self.create_operation_node(node)
180
+
181
+ if node.op == 'in':
182
+ setattr(node.left, "argument", "operand")
183
+ setattr(node.right, "argument", "set")
184
+ elif node.op == 'match':
185
+ setattr(node.left, "argument", "operand")
186
+ setattr(node.right, "argument", "pattern")
187
+ else:
188
+ setattr(node.left, "argument", "left")
189
+ setattr(node.right, "argument", "right")
190
+ setattr(node.left, "parent", operand_node)
191
+ setattr(node.right, "parent", operand_node)
192
+ self.visit(node.left)
193
+ self.visit(node.right)
194
+
195
+ def visit_UnaryOp(self, node: UnaryOp):
196
+ if node.op == '+':
197
+ setattr(node, "operator_name", "Unary plus")
198
+ elif node.op == '-':
199
+ setattr(node, "operator_name", "Unary minus")
200
+ operand_node = self.create_operation_node(node)
201
+ setattr(node.operand, "argument", "operand")
202
+ setattr(node.operand, "parent", operand_node)
203
+ self.visit(node.operand)
204
+
205
+ def visit_CondExpr(self, node: CondExpr):
206
+ setattr(node, "op", "if-then-else")
207
+ operand_if = self.create_operation_node(node)
208
+ setattr(node.condition, "parent", operand_if)
209
+ setattr(node.condition, "argument", "condition")
210
+ self.visit(node.condition)
211
+ setattr(node.then_expr, "argument", "then")
212
+ setattr(node.then_expr, "parent", operand_if)
213
+ self.visit(node.then_expr)
214
+ if node.else_expr:
215
+ setattr(node.else_expr, "argument", "else")
216
+ setattr(node.else_expr, "parent", operand_if)
217
+ self.visit(node.else_expr)
218
+
219
+ def visit_WithExpression(self, node: WithExpression):
220
+ parent = getattr(node, 'parent', None)
221
+ if parent:
222
+ setattr(node.expression, "parent", parent)
223
+ setattr(node.expression, 'argument', getattr(node, 'argument', None))
224
+ self.visit(node.expression)
225
+
226
+ def visit_AggregationOp(self, node: AggregationOp):
227
+ operand_node = self.create_operation_node(node)
228
+ setattr(node.operand, "parent", operand_node)
229
+ setattr(node.operand, "argument", "operand")
230
+ self.visit(node.operand)
231
+ if node.grouping_clause:
232
+ setattr(node.grouping_clause, "op", "group by")
233
+ setattr(node.grouping_clause, "parent", operand_node)
234
+ setattr(node.grouping_clause, "argument", "grouping_clause")
235
+ self.visit(node.grouping_clause)
236
+
237
+ def visit_GroupingClause(self, node: GroupingClause):
238
+ gc_node = self.create_operation_node(node)
239
+ for component in node.components:
240
+ element = AST()
241
+ setattr(element, "parent", gc_node)
242
+ setattr(element, "argument", "component")
243
+ comp_node = self.create_operation_node(element, is_leaf=True)
244
+ # property_id = ItemCategory.get_property_id_from_code(code=node.component, session=self.session)[0]
245
+ if component in ('r', 'c', 's'):
246
+ op_ref = OperandReference(
247
+ op_node=comp_node,
248
+ OperandReference=component
249
+ )
250
+ else:
251
+ property_id = ItemCategory.get_property_id_from_code(code=component, session=self.session_queries)[0]
252
+ op_ref = OperandReference(
253
+ op_node=comp_node,
254
+ OperandReference='property',
255
+ PropertyID=property_id
256
+ )
257
+
258
+ self.session.add(op_ref)
259
+
260
+ def visit_ComplexNumericOp(self, node: ComplexNumericOp):
261
+ operand_node = self.create_operation_node(node)
262
+ for operand in node.operands:
263
+ setattr(operand, "parent", operand_node)
264
+ setattr(operand, "argument", "operand")
265
+ self.visit(operand)
266
+
267
+ def visit_FilterOp(self, node: FilterOp):
268
+ setattr(node, "op", "filter")
269
+ operand_node = self.create_operation_node(node)
270
+ setattr(node.selection, "parent", operand_node)
271
+ setattr(node.selection, "argument", "selection")
272
+
273
+ setattr(node.condition, "parent", operand_node)
274
+ setattr(node.condition, "argument", "condition")
275
+
276
+ self.visit(node.selection)
277
+ self.visit(node.condition)
278
+
279
+ def visit_TimeShiftOp(self, node: TimeShiftOp):
280
+ setattr(node, "op", "time_shift")
281
+ get_node = self.create_operation_node(node)
282
+
283
+ setattr(node.operand, "parent", get_node)
284
+ setattr(node.operand, "argument", "operand")
285
+ self.visit(node.operand)
286
+
287
+ # period indicator
288
+ period_indicator_node = AST()
289
+ setattr(period_indicator_node, "parent", get_node)
290
+ setattr(period_indicator_node, "argument", "period_indicator")
291
+ setattr(period_indicator_node, "scalar", "Q")
292
+ self.create_operation_node(period_indicator_node, is_leaf=True)
293
+
294
+ # shift number
295
+ shift_number_node = AST()
296
+ setattr(shift_number_node, "parent", get_node)
297
+ setattr(shift_number_node, "argument", "shift_number")
298
+ setattr(shift_number_node, "scalar", getattr(node, 'shift_number', None))
299
+ self.create_operation_node(shift_number_node, is_leaf=True)
300
+
301
+ # component
302
+ ast_element = AST()
303
+ setattr(ast_element, "parent", get_node)
304
+ setattr(ast_element, "argument", "dimension")
305
+ setattr(ast_element, "source_reference", "property")
306
+ operand_node = self.create_operation_node(ast_element, is_leaf=True) # TODO: Adapt to refPeriod
307
+ if property_ref_period_mangement(node.component):
308
+ op_ref = OperandReference(
309
+ op_node=operand_node,
310
+ OperandReference="refPeriod"
311
+ )
312
+ else:
313
+ property_id = ItemCategory.get_property_id_from_code(code=node.component, session=self.session_queries)[0]
314
+ op_ref = OperandReference(
315
+ op_node=operand_node,
316
+ OperandReference=getattr(ast_element, "source_reference"),
317
+ PropertyID=property_id
318
+ )
319
+ self.session.add(op_ref)
320
+
321
+ def visit_WhereClauseOp(self, node: WhereClauseOp):
322
+ setattr(node, "op", "where")
323
+ node_op = self.create_operation_node(node)
324
+ setattr(node.operand, "parent", node_op)
325
+ setattr(node.operand, "argument", "operand")
326
+
327
+ setattr(node.condition, "parent", node_op)
328
+ setattr(node.condition, "argument", "condition")
329
+ self.visit(node.operand)
330
+ self.visit(node.condition)
331
+
332
+ def visit_GetOp(self, node: GetOp):
333
+ setattr(node, "op", "get")
334
+ node_op = self.create_operation_node(node)
335
+ setattr(node.operand, "parent", node_op)
336
+ setattr(node.operand, "argument", "operand")
337
+ self.visit(node.operand)
338
+
339
+ element = AST()
340
+ setattr(element, "parent", node_op)
341
+ setattr(element, "argument", "component")
342
+ setattr(element, "source_reference", "property")
343
+ component_node = self.create_operation_node(element, is_leaf=True)
344
+ if property_ref_period_mangement(node.component):
345
+ op_ref = OperandReference(
346
+ op_node=component_node,
347
+ OperandReference="refPeriod"
348
+ )
349
+ else:
350
+ property_id = ItemCategory.get_property_id_from_code(code=node.component, session=self.session_queries)[0]
351
+ op_ref = OperandReference(
352
+ op_node=component_node,
353
+ OperandReference=getattr(element, "source_reference"),
354
+ PropertyID=property_id
355
+ )
356
+ self.session.add(op_ref)
357
+
358
+ def visit_RenameOp(self, node: RenameOp):
359
+ setattr(node, "op", "rename")
360
+ operand_node = self.create_operation_node(node)
361
+ setattr(node.operand, "parent", operand_node)
362
+ setattr(node.operand, "argument", "operand")
363
+ self.visit(node.operand)
364
+
365
+ for rename_node in node.rename_nodes:
366
+ setattr(rename_node, "parent", operand_node)
367
+ setattr(rename_node, "argument", "node")
368
+ self.visit(rename_node)
369
+
370
+ def visit_RenameNode(self, node: RenameNode):
371
+ setattr(node, "op", "node")
372
+ rename_node = self.create_operation_node(node)
373
+ old_name = AST()
374
+ setattr(old_name, "parent", rename_node)
375
+ setattr(old_name, "argument", "old_name")
376
+ setattr(old_name, "source_reference", "property")
377
+ setattr(old_name, "scalar", node.old_name)
378
+ old_property_id = ItemCategory.get_property_id_from_code(code=node.old_name, session=self.session_queries)[0]
379
+ old_name_node = self.create_operation_node(old_name, is_leaf=True)
380
+
381
+ old_operand_ref = OperandReference(
382
+ op_node=old_name_node,
383
+ OperandReference=getattr(old_name, "source_reference"),
384
+ PropertyID=old_property_id
385
+ )
386
+
387
+ self.session.add(old_operand_ref)
388
+
389
+ new_name = AST()
390
+ setattr(new_name, "parent", rename_node)
391
+ setattr(new_name, "argument", "new_name")
392
+ setattr(new_name, "source_reference", "property")
393
+ setattr(new_name, "scalar", node.new_name)
394
+ new_name_node = self.create_operation_node(new_name, is_leaf=True)
395
+
396
+ new_operand_ref = OperandReference(
397
+ op_node=new_name_node,
398
+ OperandReference=getattr(new_name, "source_reference"),
399
+ PropertyID=old_property_id
400
+ )
401
+ self.session.add(new_operand_ref)
402
+
403
+ def visit_PreconditionItem(self, node: PreconditionItem):
404
+ operand_node = self.create_operation_node(node, is_leaf=True)
405
+ operand_reference = "PreconditionItem" # "$_{}".format(node.value)
406
+ precondition_var = VariableVersion.check_precondition(self.session, node.variable_code, self.release_id)
407
+ variable_id = None
408
+ precondition_code = None
409
+ if precondition_var:
410
+ variable_id = precondition_var.VariableID
411
+ precondition_code = precondition_var.Code
412
+ else:
413
+ preconditions_vars = VariableVersion.get_all_preconditions(self.session, self.release_id)
414
+ precondition_found = False
415
+ for precondition in preconditions_vars:
416
+ if precondition.Code in node.variable_code:
417
+ variable_id = precondition.VariableID
418
+ precondition_found = True
419
+ precondition_code = precondition.Code
420
+ break
421
+
422
+ if not precondition_found:
423
+ raise SemanticError("1-3", variable=node.variable_code)
424
+
425
+ operand_ref = OperandReference(
426
+ op_node=operand_node,
427
+ OperandReference=operand_reference,
428
+ VariableID=variable_id
429
+ )
430
+
431
+ self.session.add(operand_ref)
432
+ self.precondition_items.append(precondition_code)
433
+
434
+ def visit_VarRef(self, node: VarRef):
435
+ setattr(node, "source_reference", "variable")
436
+ op_node = self.create_operation_node(node, is_leaf=True)
437
+ node_value = getattr(node, "value", getattr(node, "variable", None))
438
+ variable_id = VariableVersion.get_VariableID(self.session, node_value, self.release_id)
439
+ if variable_id:
440
+ variable_id = variable_id[0]
441
+ else:
442
+ raise SemanticError("1-3", variable=node_value)
443
+ operand_ref = OperandReference(
444
+ op_node=op_node,
445
+ OperandReference=getattr(node, "source_reference"),
446
+ VariableID=variable_id
447
+ )
448
+
449
+ self.session.add(operand_ref)
450
+
451
+ def visit_VarID(self, node: VarID):
452
+ setattr(node, "source_reference", "variable")
453
+
454
+ op_node = self.create_operation_node(node, is_leaf=True)
455
+
456
+ data_xyz = self.extract_operand_data(node.table, node.rows, node.cols, node.sheets)
457
+
458
+ # Extracting data
459
+ significant_rows = node.rows is not None and len(node.rows) >= 1
460
+ significant_cols = node.cols is not None and len(node.cols) >= 1
461
+ significant_sheets = node.sheets is not None and len(node.sheets) >= 1
462
+
463
+ for e in data_xyz:
464
+ operand_ref = OperandReference(
465
+ op_node=op_node,
466
+ x=e['x'] if significant_rows else None,
467
+ y=e['y'] if significant_cols else None,
468
+ z=e['z'] if significant_sheets else None,
469
+ OperandReference=getattr(node, "source_reference"),
470
+ VariableID=e['variable_id']
471
+ )
472
+
473
+ self.session.add(operand_ref)
474
+
475
+ operand_ref_loc = OperandReferenceLocation(
476
+ op_reference=operand_ref,
477
+ CellID=e['cell_id'],
478
+ Table=node.table,
479
+ Row=e['row_code'],
480
+ column=e['column_code'],
481
+ Sheet=e['sheet_code']
482
+ )
483
+
484
+ self.session.add(operand_ref_loc)
485
+
486
+ if node.table not in self.table_vid_dict:
487
+ table_vid = int(self.data[self.data['table_code'] == node.table]['table_vid'].unique()[0])
488
+ self.table_vid_dict[node.table] = table_vid
489
+
490
+ if self.is_scripting:
491
+ self._add_table_vid_to_operation_tables(node.table)
492
+
493
+ def visit_Constant(self, node: Constant):
494
+ setattr(node, "scalar", node.value)
495
+ self.create_operation_node(node, is_leaf=True)
496
+
497
+ def visit_Dimension(self, node: Dimension):
498
+ setattr(node, "source_reference", "property")
499
+ op_node = self.create_operation_node(node, is_leaf=True)
500
+ property = ItemCategory.get_property_from_code(code=node.dimension_code, session=self.session_queries)
501
+ operand_ref = OperandReference(
502
+ op_node=op_node,
503
+ OperandReference=getattr(node, "source_reference"),
504
+ PropertyID=property.ItemID
505
+ )
506
+ self.session.add(operand_ref)
507
+
508
+ def visit_Set(self, node: Set):
509
+ setattr(node, "source_reference", "item")
510
+ setattr(node, "operand_type", "set")
511
+ op_node = self.create_operation_node(node, is_leaf=True)
512
+
513
+ for child in node.children:
514
+ item_id = ItemCategory.get_item_category_id_from_signature(
515
+ signature=child.item, session=self.session_queries
516
+ )[0]
517
+ operand_ref = OperandReference(
518
+ op_node=op_node,
519
+ OperandReference=getattr(node, "source_reference"),
520
+ ItemID=item_id
521
+ )
522
+ self.session.add(operand_ref)
523
+
524
+ def visit_Scalar(self, node: Scalar):
525
+ setattr(node, "source_reference", "item")
526
+ op_node = self.create_operation_node(node, is_leaf=True)
527
+ item_id = ItemCategory.get_item_category_id_from_signature(
528
+ signature=node.item, session=self.session_queries
529
+ )[0]
530
+ operand_ref = OperandReference(
531
+ op_node=op_node,
532
+ OperandReference=getattr(node, "source_reference"),
533
+ ItemID=item_id
534
+ )
535
+ self.session.add(operand_ref)
536
+
537
+ def visit_OperationRef(self, node: OperationRef):
538
+ setattr(node, "source_reference", "operation")
539
+ op_node = self.create_operation_node(node, is_leaf=True)
540
+
541
+ op_version_id = self._get_op_version_id(node.operation_code)
542
+
543
+ operand_ref = OperandReference(
544
+ op_node=op_node,
545
+ OperandReference=op_version_id
546
+ )
547
+ self.session.add(operand_ref)
548
+
549
+ def _get_op_version_id(self, operation_code):
550
+ op_version_id = self.operations_data[self.operations_data['Code'] == operation_code]['OperationVID'].values[0]
551
+ op_version_id = int(op_version_id)
552
+ return op_version_id
553
+
554
+ def _set_op_version_id_from_operation(self, child):
555
+
556
+ operation_code = child.left.value
557
+ op_version_id = self._get_op_version_id(operation_code)
558
+ self.op_version_id = op_version_id
559
+
560
+ def _add_table_vid_to_operation_tables(self, table_code):
561
+ if self.op_version_id not in self.operation_tables:
562
+ self.operation_tables[self.op_version_id] = []
563
+ table_vid = int(self.data[self.data['table_code'] == table_code]['table_vid'].unique()[0])
564
+ if table_vid not in self.operation_tables[self.op_version_id]:
565
+ self.operation_tables[self.op_version_id].append(table_vid)
566
+
567
+ def store_objects_as_json(self):
568
+ operation_nodes = [o for o in self.session.new if isinstance(o, OperationNode) and not o.parent]
569
+ self.result['operation_nodes'] = operation_nodes
570
+ self.result['operation_scopes'] = {}
571
+ self.result['operation_scopes']['new'] = self.new_scopes
572
+ self.result['operation_scopes']['existing'] = self.existing_scopes
573
+ self.session.expunge_all()
574
+
575
+ def compare_ast(self, reference: OperationNode):
576
+ """
577
+ Compares the ML generated by the AST of the expression provided with the ML generated by the AST generated with the ML stored in the db.
578
+ :return: True if the ASTs are equal, False otherwise.
579
+ """
580
+ op_nodes = self.result['operation_nodes']
581
+
582
+ if len(op_nodes) == 0:
583
+ raise Exception("No AST Generated")
584
+
585
+ if op_nodes[0] != reference:
586
+ return False #is_same_ast = False
587
+
588
+ return True #is_same_ast = True
@@ -0,0 +1,79 @@
1
+ from py_dpm.AST.ASTObjects import Start, VarID, WithExpression
2
+ from py_dpm.AST.ASTTemplate import ASTTemplate
3
+ from py_dpm.models import ViewModules
4
+ from py_dpm.Utils.operands_mapping import LabelHandler
5
+ from py_dpm.Utils.tokens import CROSS_MODULE, INTRA_MODULE, REPEATED_INTRA_MODULE
6
+
7
+
8
+ class ModuleAnalyzer(ASTTemplate):
9
+ def __init__(self, session):
10
+
11
+ super(ASTTemplate).__init__()
12
+ self.modules = []
13
+ self.session = session
14
+ self.mode = None
15
+ self.module_info = {}
16
+ LabelHandler().reset_instance()
17
+
18
+ def new_label(self):
19
+ return LabelHandler().labels.__next__()
20
+
21
+ def extract_modules(self, tables):
22
+ return ViewModules().get_modules(self.session, tables)
23
+
24
+ def module_analysis(self):
25
+ unique_modules = []
26
+
27
+ for operand_info in self.module_info.values():
28
+ if operand_info == 'Module not found':
29
+ print(f"Module not found: {self.module_info}")
30
+ return
31
+ unique_modules += operand_info
32
+ unique_modules = list(set(unique_modules))
33
+ if len(unique_modules) == 1:
34
+ self.mode = INTRA_MODULE
35
+ self.modules = unique_modules
36
+ self.find_common_modules(unique_modules)
37
+
38
+ def visit_Start(self, node: Start):
39
+ self.visit(node.children[0])
40
+ if not isinstance(node.children[0], WithExpression):
41
+ self.module_analysis()
42
+ return self.mode, self.modules
43
+
44
+ def visit_WithExpression(self, node: WithExpression):
45
+ if node.partial_selection.table is not None:
46
+ modules = self.extract_modules([node.partial_selection.table])
47
+ self.modules = modules
48
+ if len(modules) > 1:
49
+ self.mode = REPEATED_INTRA_MODULE
50
+ elif len(modules) == 1:
51
+ self.mode = INTRA_MODULE
52
+ return
53
+ self.visit(node.expression)
54
+ self.module_analysis()
55
+
56
+ def visit_VarID(self, node: VarID):
57
+ modules = self.extract_modules([node.table])
58
+ if len(modules) > 0:
59
+ self.module_info[self.new_label()] = modules
60
+ else:
61
+ self.module_info[self.new_label()] = 'Module not found'
62
+
63
+ def find_common_modules(self, unique_modules):
64
+ common_modules = []
65
+ for operand_info in self.module_info.values():
66
+ if len(common_modules) == 0:
67
+ common_modules = operand_info
68
+ continue
69
+ common_modules = list(set(common_modules) & set(operand_info))
70
+ if len(common_modules) == 0:
71
+ if len(unique_modules) > 1:
72
+ self.mode = CROSS_MODULE
73
+ self.modules = unique_modules
74
+ return
75
+ elif len(common_modules) == 1:
76
+ self.mode = INTRA_MODULE
77
+ else:
78
+ self.mode = CROSS_MODULE
79
+ self.modules = common_modules