pydpm_xl 0.1.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- py_dpm/AST/ASTConstructor.py +503 -0
- py_dpm/AST/ASTObjects.py +827 -0
- py_dpm/AST/ASTTemplate.py +101 -0
- py_dpm/AST/ASTVisitor.py +13 -0
- py_dpm/AST/MLGeneration.py +588 -0
- py_dpm/AST/ModuleAnalyzer.py +79 -0
- py_dpm/AST/ModuleDependencies.py +203 -0
- py_dpm/AST/WhereClauseChecker.py +12 -0
- py_dpm/AST/__init__.py +0 -0
- py_dpm/AST/check_operands.py +302 -0
- py_dpm/DataTypes/ScalarTypes.py +324 -0
- py_dpm/DataTypes/TimeClasses.py +370 -0
- py_dpm/DataTypes/TypePromotion.py +195 -0
- py_dpm/DataTypes/__init__.py +0 -0
- py_dpm/Exceptions/__init__.py +0 -0
- py_dpm/Exceptions/exceptions.py +84 -0
- py_dpm/Exceptions/messages.py +114 -0
- py_dpm/OperationScopes/OperationScopeService.py +247 -0
- py_dpm/OperationScopes/__init__.py +0 -0
- py_dpm/Operators/AggregateOperators.py +138 -0
- py_dpm/Operators/BooleanOperators.py +30 -0
- py_dpm/Operators/ClauseOperators.py +159 -0
- py_dpm/Operators/ComparisonOperators.py +69 -0
- py_dpm/Operators/ConditionalOperators.py +362 -0
- py_dpm/Operators/NumericOperators.py +101 -0
- py_dpm/Operators/Operator.py +388 -0
- py_dpm/Operators/StringOperators.py +27 -0
- py_dpm/Operators/TimeOperators.py +53 -0
- py_dpm/Operators/__init__.py +0 -0
- py_dpm/Utils/ValidationsGenerationUtils.py +429 -0
- py_dpm/Utils/__init__.py +0 -0
- py_dpm/Utils/operands_mapping.py +73 -0
- py_dpm/Utils/operator_mapping.py +89 -0
- py_dpm/Utils/tokens.py +172 -0
- py_dpm/Utils/utils.py +2 -0
- py_dpm/ValidationsGeneration/PropertiesConstraintsProcessor.py +190 -0
- py_dpm/ValidationsGeneration/Utils.py +364 -0
- py_dpm/ValidationsGeneration/VariantsProcessor.py +265 -0
- py_dpm/ValidationsGeneration/__init__.py +0 -0
- py_dpm/ValidationsGeneration/auxiliary_functions.py +98 -0
- py_dpm/__init__.py +61 -0
- py_dpm/api/__init__.py +140 -0
- py_dpm/api/ast_generator.py +438 -0
- py_dpm/api/complete_ast.py +241 -0
- py_dpm/api/data_dictionary_validation.py +577 -0
- py_dpm/api/migration.py +77 -0
- py_dpm/api/semantic.py +224 -0
- py_dpm/api/syntax.py +182 -0
- py_dpm/client.py +106 -0
- py_dpm/data_handlers.py +99 -0
- py_dpm/db_utils.py +117 -0
- py_dpm/grammar/__init__.py +0 -0
- py_dpm/grammar/dist/__init__.py +0 -0
- py_dpm/grammar/dist/dpm_xlLexer.interp +428 -0
- py_dpm/grammar/dist/dpm_xlLexer.py +804 -0
- py_dpm/grammar/dist/dpm_xlLexer.tokens +106 -0
- py_dpm/grammar/dist/dpm_xlParser.interp +249 -0
- py_dpm/grammar/dist/dpm_xlParser.py +5224 -0
- py_dpm/grammar/dist/dpm_xlParser.tokens +106 -0
- py_dpm/grammar/dist/dpm_xlParserListener.py +742 -0
- py_dpm/grammar/dist/dpm_xlParserVisitor.py +419 -0
- py_dpm/grammar/dist/listeners.py +10 -0
- py_dpm/grammar/dpm_xlLexer.g4 +435 -0
- py_dpm/grammar/dpm_xlParser.g4 +260 -0
- py_dpm/migration.py +282 -0
- py_dpm/models.py +2139 -0
- py_dpm/semantics/DAG/DAGAnalyzer.py +158 -0
- py_dpm/semantics/DAG/__init__.py +0 -0
- py_dpm/semantics/SemanticAnalyzer.py +320 -0
- py_dpm/semantics/Symbols.py +223 -0
- py_dpm/semantics/__init__.py +0 -0
- py_dpm/utils/__init__.py +0 -0
- py_dpm/utils/ast_serialization.py +481 -0
- py_dpm/views/data_types.sql +12 -0
- py_dpm/views/datapoints.sql +65 -0
- py_dpm/views/hierarchy_operand_reference.sql +11 -0
- py_dpm/views/hierarchy_preconditions.sql +13 -0
- py_dpm/views/hierarchy_variables.sql +26 -0
- py_dpm/views/hierarchy_variables_context.sql +14 -0
- py_dpm/views/key_components.sql +18 -0
- py_dpm/views/module_from_table.sql +11 -0
- py_dpm/views/open_keys.sql +13 -0
- py_dpm/views/operation_info.sql +27 -0
- py_dpm/views/operation_list.sql +18 -0
- py_dpm/views/operations_versions_from_module_version.sql +30 -0
- py_dpm/views/precondition_info.sql +17 -0
- py_dpm/views/report_type_operand_reference_info.sql +18 -0
- py_dpm/views/subcategory_info.sql +17 -0
- py_dpm/views/table_info.sql +19 -0
- pydpm_xl-0.1.10.dist-info/LICENSE +674 -0
- pydpm_xl-0.1.10.dist-info/METADATA +50 -0
- pydpm_xl-0.1.10.dist-info/RECORD +94 -0
- pydpm_xl-0.1.10.dist-info/WHEEL +4 -0
- pydpm_xl-0.1.10.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import networkx as nx
|
|
3
|
+
|
|
4
|
+
from py_dpm.AST.ASTObjects import Start, PersistentAssignment, WithExpression, AST, OperationRef, TemporaryAssignment
|
|
5
|
+
from py_dpm.AST.ASTTemplate import ASTTemplate
|
|
6
|
+
from py_dpm.Exceptions import exceptions
|
|
7
|
+
from py_dpm.Utils.tokens import INPUTS, OUTPUTS
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class DAGAnalyzer(ASTTemplate):
|
|
11
|
+
"""
|
|
12
|
+
Class to generate the Direct Acyclic Graph from calculations scripts.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
def __init__(self):
|
|
16
|
+
super().__init__()
|
|
17
|
+
self.inputs: list = []
|
|
18
|
+
self.outputs: list = []
|
|
19
|
+
self.dependencies: dict = {}
|
|
20
|
+
self.calculation_number = 1
|
|
21
|
+
|
|
22
|
+
self.vertex: dict = {}
|
|
23
|
+
self.edges: dict = {}
|
|
24
|
+
self.number_of_vertex: int = 0
|
|
25
|
+
self.adjacency_matrix = None
|
|
26
|
+
self.sorting = None
|
|
27
|
+
|
|
28
|
+
self.graph = None
|
|
29
|
+
|
|
30
|
+
def create_DAG(self, ast: AST):
|
|
31
|
+
"""
|
|
32
|
+
Method to generate the Direct Acyclic Graph from ast
|
|
33
|
+
"""
|
|
34
|
+
self.visit(ast)
|
|
35
|
+
|
|
36
|
+
self.load_vertex()
|
|
37
|
+
self.load_edges()
|
|
38
|
+
self.create_adjacency_matrix()
|
|
39
|
+
|
|
40
|
+
try:
|
|
41
|
+
self.nx_topological_sort()
|
|
42
|
+
if len(self.edges):
|
|
43
|
+
self.sort_ast(ast=ast)
|
|
44
|
+
|
|
45
|
+
except nx.NetworkXUnfeasible:
|
|
46
|
+
graph_cycles = nx.find_cycle(self.graph)
|
|
47
|
+
pos1, pos2 = graph_cycles[0]
|
|
48
|
+
op1 = self.vertex[pos1]
|
|
49
|
+
op2 = self.vertex[pos2]
|
|
50
|
+
raise exceptions.ScriptingError(code="6-4", op1=op1, op2=op2)
|
|
51
|
+
|
|
52
|
+
def load_vertex(self):
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
for key, calculation in self.dependencies.items():
|
|
58
|
+
outputs = calculation[OUTPUTS]
|
|
59
|
+
if len(outputs) > 0:
|
|
60
|
+
self.vertex[key] = outputs[0]
|
|
61
|
+
self.number_of_vertex = len(self.vertex)
|
|
62
|
+
|
|
63
|
+
def load_edges(self):
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
"""
|
|
67
|
+
if len(self.vertex) > 0:
|
|
68
|
+
number_of_edges = 0
|
|
69
|
+
for key, calculation in self.dependencies.items():
|
|
70
|
+
outputs = calculation[OUTPUTS]
|
|
71
|
+
if len(outputs) > 0:
|
|
72
|
+
output = outputs[0]
|
|
73
|
+
for sub_key, sub_calculation in self.dependencies.items():
|
|
74
|
+
inputs = sub_calculation[INPUTS]
|
|
75
|
+
if inputs and output in inputs:
|
|
76
|
+
self.edges[number_of_edges] = (key, sub_key)
|
|
77
|
+
number_of_edges += 1
|
|
78
|
+
|
|
79
|
+
def create_adjacency_matrix(self):
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
"""
|
|
83
|
+
self.adjacency_matrix = np.zeros((self.number_of_vertex, self.number_of_vertex), dtype=int)
|
|
84
|
+
for edge in list(self.edges.values()):
|
|
85
|
+
self.adjacency_matrix[edge[0] - 1][edge[1] - 1] = 1
|
|
86
|
+
|
|
87
|
+
def nx_topological_sort(self):
|
|
88
|
+
"""
|
|
89
|
+
|
|
90
|
+
"""
|
|
91
|
+
edges = list(self.edges.values())
|
|
92
|
+
self.graph = DAG = nx.DiGraph()
|
|
93
|
+
DAG.add_nodes_from(self.vertex)
|
|
94
|
+
DAG.add_edges_from(edges)
|
|
95
|
+
self.sorting = list(nx.topological_sort(DAG))
|
|
96
|
+
|
|
97
|
+
def sort_ast(self, ast):
|
|
98
|
+
"""
|
|
99
|
+
|
|
100
|
+
"""
|
|
101
|
+
lst = []
|
|
102
|
+
calculations = list(ast.children)
|
|
103
|
+
for x in self.sorting:
|
|
104
|
+
for i in range(len(calculations)):
|
|
105
|
+
if i == x - 1:
|
|
106
|
+
lst.append(calculations[i])
|
|
107
|
+
self.check_overwriting(lst)
|
|
108
|
+
ast.children = lst
|
|
109
|
+
|
|
110
|
+
def check_overwriting(self, outputs):
|
|
111
|
+
"""
|
|
112
|
+
|
|
113
|
+
"""
|
|
114
|
+
non_repeated = []
|
|
115
|
+
outputs_lst = []
|
|
116
|
+
for output in outputs:
|
|
117
|
+
if isinstance(output, TemporaryAssignment):
|
|
118
|
+
temporary_identifier_value = output.left.value
|
|
119
|
+
outputs_lst.append(temporary_identifier_value)
|
|
120
|
+
if isinstance(output.right, PersistentAssignment):
|
|
121
|
+
outputs_lst.append(output.right.left.variable)
|
|
122
|
+
for output_value in outputs_lst:
|
|
123
|
+
if output_value not in non_repeated:
|
|
124
|
+
non_repeated.append(output_value)
|
|
125
|
+
else:
|
|
126
|
+
raise exceptions.SemanticError("6-1", variable=output_value)
|
|
127
|
+
|
|
128
|
+
def get_calculation_structure(self):
|
|
129
|
+
"""
|
|
130
|
+
|
|
131
|
+
"""
|
|
132
|
+
inputs = list(set(self.inputs))
|
|
133
|
+
outputs = list(set(self.outputs))
|
|
134
|
+
|
|
135
|
+
return {INPUTS: inputs, OUTPUTS: outputs}
|
|
136
|
+
|
|
137
|
+
def visit_Start(self, node: Start):
|
|
138
|
+
for child in node.children:
|
|
139
|
+
self.visit(child)
|
|
140
|
+
self.dependencies[self.calculation_number] = self.get_calculation_structure()
|
|
141
|
+
|
|
142
|
+
self.calculation_number += 1
|
|
143
|
+
|
|
144
|
+
self.inputs = []
|
|
145
|
+
self.outputs = []
|
|
146
|
+
|
|
147
|
+
def visit_PersistentAssignment(self, node: PersistentAssignment):
|
|
148
|
+
self.visit(node.right)
|
|
149
|
+
|
|
150
|
+
def visit_TemporaryAssignment(self, node: TemporaryAssignment):
|
|
151
|
+
self.outputs.append(node.left.value)
|
|
152
|
+
self.visit(node.right)
|
|
153
|
+
|
|
154
|
+
def visit_OperationRef(self, node: OperationRef):
|
|
155
|
+
self.inputs.append(node.operation_code)
|
|
156
|
+
|
|
157
|
+
def visit_WithExpression(self, node: WithExpression):
|
|
158
|
+
self.visit(node.expression)
|
|
File without changes
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
from abc import ABC
|
|
3
|
+
|
|
4
|
+
import pandas as pd
|
|
5
|
+
|
|
6
|
+
from py_dpm.AST.ASTObjects import AggregationOp, BinOp, ComplexNumericOp, CondExpr, Constant, \
|
|
7
|
+
Dimension, FilterOp, GetOp, OperationRef, \
|
|
8
|
+
ParExpr, PersistentAssignment, PreconditionItem, PropertyReference, RenameOp, \
|
|
9
|
+
Scalar as ScalarNode, Set, Start, TemporaryAssignment, \
|
|
10
|
+
TimeShiftOp, UnaryOp, VarID, VarRef, WhereClauseOp, WithExpression
|
|
11
|
+
from py_dpm.AST.ASTTemplate import ASTTemplate
|
|
12
|
+
from py_dpm.DataTypes.ScalarTypes import Item, Mixed, Null, ScalarFactory
|
|
13
|
+
from py_dpm.DataTypes.TypePromotion import binary_implicit_type_promotion
|
|
14
|
+
from py_dpm.Exceptions import exceptions
|
|
15
|
+
from py_dpm.Exceptions.exceptions import SemanticError
|
|
16
|
+
from py_dpm.Utils.operands_mapping import set_operand_label
|
|
17
|
+
from py_dpm.Utils.operator_mapping import AGGR_OP_MAPPING, BIN_OP_MAPPING, CLAUSE_OP_MAPPING, \
|
|
18
|
+
COMPLEX_OP_MAPPING, CONDITIONAL_OP_MAPPING, \
|
|
19
|
+
TIME_OPERATORS, UNARY_OP_MAPPING
|
|
20
|
+
from py_dpm.Utils.tokens import DPM, FILTER, GET, IF, RENAME, STANDARD, TIME_SHIFT, WHERE
|
|
21
|
+
from py_dpm.data_handlers import filter_all_data
|
|
22
|
+
from py_dpm.semantics.Symbols import ConstantOperand, FactComponent, KeyComponent, RecordSet, \
|
|
23
|
+
Scalar, \
|
|
24
|
+
ScalarSet, Structure
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class InputAnalyzer(ASTTemplate, ABC):
|
|
28
|
+
|
|
29
|
+
def __init__(self, expression):
|
|
30
|
+
super().__init__()
|
|
31
|
+
self.data = None
|
|
32
|
+
self.key_components = {}
|
|
33
|
+
self.open_keys = None
|
|
34
|
+
self.result: bool = False
|
|
35
|
+
self._expression: str = expression # For debugging purposes only
|
|
36
|
+
self.preconditions: bool = False
|
|
37
|
+
|
|
38
|
+
self.calculations_outputs = {}
|
|
39
|
+
|
|
40
|
+
self.global_variables = {'refPeriod': ScalarFactory().database_types_mapping('d')()}
|
|
41
|
+
|
|
42
|
+
# Start of visiting nodes.
|
|
43
|
+
|
|
44
|
+
def visit_Start(self, node: Start):
|
|
45
|
+
|
|
46
|
+
result = []
|
|
47
|
+
for child in node.children:
|
|
48
|
+
result_symbol = self.visit(child)
|
|
49
|
+
|
|
50
|
+
if self.preconditions:
|
|
51
|
+
if isinstance(result_symbol, Scalar) and not result_symbol.type.strictly_same_class(ScalarFactory().scalar_factory("Boolean")):
|
|
52
|
+
raise exceptions.SemanticError("2-1")
|
|
53
|
+
elif isinstance(result_symbol, RecordSet):
|
|
54
|
+
if (not result_symbol.has_only_global_components) or (
|
|
55
|
+
not result_symbol.get_fact_component().type.strictly_same_class(
|
|
56
|
+
ScalarFactory().scalar_factory("Boolean"))):
|
|
57
|
+
raise exceptions.SemanticError("2-1")
|
|
58
|
+
|
|
59
|
+
if len(node.children) == 1:
|
|
60
|
+
return result_symbol
|
|
61
|
+
result.append(result_symbol)
|
|
62
|
+
|
|
63
|
+
return result
|
|
64
|
+
|
|
65
|
+
def visit_PersistentAssignment(self, node: PersistentAssignment):
|
|
66
|
+
return self.visit(node.right)
|
|
67
|
+
|
|
68
|
+
def visit_TemporaryAssignment(self, node: TemporaryAssignment):
|
|
69
|
+
|
|
70
|
+
right = self.visit(node.right)
|
|
71
|
+
right.name = node.left.value
|
|
72
|
+
self.calculations_outputs[right.name] = right
|
|
73
|
+
return right
|
|
74
|
+
|
|
75
|
+
def visit_ParExpr(self, node: ParExpr):
|
|
76
|
+
return self.visit(node.expression)
|
|
77
|
+
|
|
78
|
+
def visit_BinOp(self, node: BinOp):
|
|
79
|
+
left_symbol = self.visit(node.left)
|
|
80
|
+
right_symbol = self.visit(node.right)
|
|
81
|
+
result = BIN_OP_MAPPING[node.op].validate(left_symbol, right_symbol)
|
|
82
|
+
|
|
83
|
+
return result
|
|
84
|
+
|
|
85
|
+
def visit_UnaryOp(self, node: UnaryOp):
|
|
86
|
+
operand_symbol = self.visit(node.operand)
|
|
87
|
+
result = UNARY_OP_MAPPING[node.op].validate_types(operand_symbol)
|
|
88
|
+
|
|
89
|
+
return result
|
|
90
|
+
|
|
91
|
+
def visit_CondExpr(self, node: CondExpr):
|
|
92
|
+
condition_symbol = self.visit(node.condition)
|
|
93
|
+
then_symbol = self.visit(node.then_expr)
|
|
94
|
+
else_symbol = None if node.else_expr is None else self.visit(node.else_expr)
|
|
95
|
+
result = CONDITIONAL_OP_MAPPING[IF].validate(condition_symbol, then_symbol, else_symbol)
|
|
96
|
+
return result
|
|
97
|
+
|
|
98
|
+
def visit_VarRef(self, node: VarRef):
|
|
99
|
+
raise SemanticError("7-2")
|
|
100
|
+
|
|
101
|
+
def visit_PropertyReference(self, node: PropertyReference):
|
|
102
|
+
raise SemanticError("7-1")
|
|
103
|
+
|
|
104
|
+
@staticmethod
|
|
105
|
+
def __check_default_value(default_value, type_):
|
|
106
|
+
if default_value is None:
|
|
107
|
+
return
|
|
108
|
+
default_type = ScalarFactory().scalar_factory(code=default_value.type)
|
|
109
|
+
try:
|
|
110
|
+
binary_implicit_type_promotion(default_type, type_)
|
|
111
|
+
except SemanticError:
|
|
112
|
+
raise exceptions.SemanticError("3-6", expected_type=type_, default_type=default_type)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def visit_VarID(self, node: VarID):
|
|
116
|
+
|
|
117
|
+
self.__check_default_value(node.default, getattr(node, 'type'))
|
|
118
|
+
|
|
119
|
+
# filter by table_code
|
|
120
|
+
df = filter_all_data(self.data, node.table, node.rows, node.cols, node.sheets)
|
|
121
|
+
|
|
122
|
+
scalar_factory = ScalarFactory()
|
|
123
|
+
interval = getattr(node, "interval", None)
|
|
124
|
+
data_types = df["data_type"].apply(lambda x: scalar_factory.from_database_to_scalar_types(x, interval))
|
|
125
|
+
df['data_type'] = data_types
|
|
126
|
+
|
|
127
|
+
label = getattr(node, "label", None)
|
|
128
|
+
components = []
|
|
129
|
+
if self.key_components and node.table in self.key_components:
|
|
130
|
+
dpm_keys = self.key_components[node.table]
|
|
131
|
+
if len(dpm_keys) > 0:
|
|
132
|
+
for key_name, key_type in zip(dpm_keys['property_code'], dpm_keys['data_type']):
|
|
133
|
+
if not key_type:
|
|
134
|
+
type_ = Item()
|
|
135
|
+
else:
|
|
136
|
+
type_ = ScalarFactory().database_types_mapping(key_type)()
|
|
137
|
+
components.append(KeyComponent(name=key_name, type_=type_, subtype=DPM, parent=label))
|
|
138
|
+
|
|
139
|
+
standard_keys = []
|
|
140
|
+
self._check_standard_key(standard_keys, df['row_code'], 'r', label)
|
|
141
|
+
self._check_standard_key(standard_keys, df['column_code'], 'c', label)
|
|
142
|
+
self._check_standard_key(standard_keys, df['sheet_code'], 's', label)
|
|
143
|
+
|
|
144
|
+
if len(self.global_variables):
|
|
145
|
+
for var_name, var_type in self.global_variables.items():
|
|
146
|
+
var_component = KeyComponent(name=var_name, type_=var_type, subtype=DPM, parent=label, is_global=True)
|
|
147
|
+
components.append(var_component)
|
|
148
|
+
|
|
149
|
+
components.extend(standard_keys)
|
|
150
|
+
if len(components) == 0:
|
|
151
|
+
set_operand_label(label=label, operand=node)
|
|
152
|
+
return Scalar(type_=getattr(node, 'type'), name=label, origin=label)
|
|
153
|
+
fact_component = FactComponent(type_=getattr(node, 'type'), parent=label)
|
|
154
|
+
|
|
155
|
+
components.append(fact_component)
|
|
156
|
+
structure = Structure(components)
|
|
157
|
+
recordset = RecordSet(structure, name=label, origin=label)
|
|
158
|
+
|
|
159
|
+
records = []
|
|
160
|
+
standard_key_names = []
|
|
161
|
+
if len(standard_keys) > 0:
|
|
162
|
+
for key in standard_keys:
|
|
163
|
+
standard_key_names.append(key.name)
|
|
164
|
+
if key.name == 'r':
|
|
165
|
+
records.append(df['row_code'])
|
|
166
|
+
elif key.name == 'c':
|
|
167
|
+
records.append(df['column_code'])
|
|
168
|
+
elif key.name == 's':
|
|
169
|
+
records.append(df['sheet_code'])
|
|
170
|
+
|
|
171
|
+
df_records = pd.concat(records, axis=1)
|
|
172
|
+
df_records.columns = standard_key_names
|
|
173
|
+
df_records['data_type'] = df['data_type']
|
|
174
|
+
|
|
175
|
+
repeated_identifiers = df_records[df_records[standard_key_names].duplicated()]
|
|
176
|
+
if len(repeated_identifiers) > 0:
|
|
177
|
+
repeated_values = ""
|
|
178
|
+
for value in repeated_identifiers.values:
|
|
179
|
+
repeated_values = ', '.join([repeated_values, str(value)]) if repeated_values else str(value)
|
|
180
|
+
raise exceptions.SemanticError("2-6", name=getattr(node, 'label', None), keys=standard_key_names,
|
|
181
|
+
values=repeated_values)
|
|
182
|
+
|
|
183
|
+
recordset.records = df_records
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
return recordset
|
|
187
|
+
|
|
188
|
+
@staticmethod
|
|
189
|
+
def _check_standard_key(key_components, elements, name, parent):
|
|
190
|
+
if len(elements) > 1 and len(elements.unique()) > 1:
|
|
191
|
+
key_component = KeyComponent(name=name, type_=Null(), subtype=STANDARD, parent=parent)
|
|
192
|
+
key_components.append(key_component)
|
|
193
|
+
|
|
194
|
+
def visit_Constant(self, node: Constant):
|
|
195
|
+
constant_type = ScalarFactory().scalar_factory(code=node.type)
|
|
196
|
+
return ConstantOperand(type_=constant_type, name=None, origin=node.value, value=node.value)
|
|
197
|
+
|
|
198
|
+
def visit_AggregationOp(self, node: AggregationOp):
|
|
199
|
+
operand = self.visit(node.operand)
|
|
200
|
+
if not isinstance(operand, RecordSet):
|
|
201
|
+
raise exceptions.SemanticError("4-4-0-1", op=node.op)
|
|
202
|
+
|
|
203
|
+
if operand.has_only_global_components:
|
|
204
|
+
warnings.warn(
|
|
205
|
+
f"Performing an aggregation on recordset: {operand.name} which has only global key components")
|
|
206
|
+
|
|
207
|
+
grouping_clause = None
|
|
208
|
+
if node.grouping_clause:
|
|
209
|
+
grouping_clause = node.grouping_clause.components
|
|
210
|
+
|
|
211
|
+
if isinstance(operand.get_fact_component().type, Mixed):
|
|
212
|
+
origin_expression =AGGR_OP_MAPPING[node.op].generate_origin_expression(operand, grouping_clause)
|
|
213
|
+
raise exceptions.SemanticError("4-4-0-3", origin=origin_expression)
|
|
214
|
+
|
|
215
|
+
result = AGGR_OP_MAPPING[node.op].validate(operand, grouping_clause)
|
|
216
|
+
return result
|
|
217
|
+
|
|
218
|
+
def visit_Dimension(self, node: Dimension):
|
|
219
|
+
dimension_data = self.open_keys[self.open_keys['property_code'] == node.dimension_code].reset_index(
|
|
220
|
+
drop=True)
|
|
221
|
+
if dimension_data['data_type'][0] is not None:
|
|
222
|
+
type_code = dimension_data['data_type'][0]
|
|
223
|
+
type_ = ScalarFactory().database_types_mapping(code=type_code)()
|
|
224
|
+
else:
|
|
225
|
+
type_ = ScalarFactory().scalar_factory(code='Item')
|
|
226
|
+
return Scalar(type_=type_, name=None, origin=node.dimension_code)
|
|
227
|
+
|
|
228
|
+
def visit_Set(self, node: Set):
|
|
229
|
+
|
|
230
|
+
if isinstance(node.children[0], Constant):
|
|
231
|
+
types = {child.type for child in node.children}
|
|
232
|
+
if len(types) > 1:
|
|
233
|
+
raise exceptions.SemanticError('11', types=', '.join(types))
|
|
234
|
+
common_type_code = types.pop()
|
|
235
|
+
origin_elements = [str(child.value) for child in node.children]
|
|
236
|
+
else:
|
|
237
|
+
common_type_code = 'Item'
|
|
238
|
+
origin_elements = ["[" + child.item + "]" for child in node.children]
|
|
239
|
+
common_type = ScalarFactory().scalar_factory(common_type_code)
|
|
240
|
+
origin = ', '.join(origin_elements)
|
|
241
|
+
origin = "{" + origin + '}'
|
|
242
|
+
|
|
243
|
+
return ScalarSet(type_=common_type, name=None, origin=origin)
|
|
244
|
+
|
|
245
|
+
def visit_Scalar(self, node: ScalarNode):
|
|
246
|
+
type_ = ScalarFactory().scalar_factory(node.scalar_type)
|
|
247
|
+
return Scalar(type_=type_, origin=node.item, name=None)
|
|
248
|
+
|
|
249
|
+
def visit_ComplexNumericOp(self, node: ComplexNumericOp):
|
|
250
|
+
if node.op not in COMPLEX_OP_MAPPING:
|
|
251
|
+
raise NotImplementedError
|
|
252
|
+
|
|
253
|
+
symbols = [self.visit(operand) for operand in node.operands]
|
|
254
|
+
|
|
255
|
+
result = COMPLEX_OP_MAPPING[node.op].validate(symbols)
|
|
256
|
+
return result
|
|
257
|
+
|
|
258
|
+
def visit_FilterOp(self, node: FilterOp):
|
|
259
|
+
selection = self.visit(node.selection)
|
|
260
|
+
condition = self.visit(node.condition)
|
|
261
|
+
result = CONDITIONAL_OP_MAPPING[FILTER].validate(selection, condition)
|
|
262
|
+
return result
|
|
263
|
+
|
|
264
|
+
def visit_TimeShiftOp(self, node: TimeShiftOp):
|
|
265
|
+
operand = self.visit(node.operand)
|
|
266
|
+
if not isinstance(operand, (RecordSet, Scalar, ConstantOperand)):
|
|
267
|
+
raise exceptions.SemanticError("4-7-1", op=TIME_SHIFT)
|
|
268
|
+
result = TIME_OPERATORS[TIME_SHIFT].validate(operand=operand, component_name=node.component,
|
|
269
|
+
period=node.period_indicator, shift_number=node.shift_number)
|
|
270
|
+
return result
|
|
271
|
+
|
|
272
|
+
def visit_WhereClauseOp(self, node: WhereClauseOp):
|
|
273
|
+
|
|
274
|
+
operand = self.visit(node.operand)
|
|
275
|
+
|
|
276
|
+
if len(node.key_components) == 0:
|
|
277
|
+
raise exceptions.SemanticError("4-5-2-1", recordset=operand.name)
|
|
278
|
+
|
|
279
|
+
condition = self.visit(node.condition)
|
|
280
|
+
result = CLAUSE_OP_MAPPING[WHERE].validate(operand=operand, key_names=node.key_components, new_names=None,
|
|
281
|
+
condition=condition)
|
|
282
|
+
return result
|
|
283
|
+
|
|
284
|
+
def visit_RenameOp(self, node: RenameOp):
|
|
285
|
+
operand = self.visit(node.operand)
|
|
286
|
+
names = []
|
|
287
|
+
new_names = []
|
|
288
|
+
for rename_node in node.rename_nodes:
|
|
289
|
+
names.append(rename_node.old_name)
|
|
290
|
+
new_names.append(rename_node.new_name)
|
|
291
|
+
result = CLAUSE_OP_MAPPING[RENAME].validate(operand=operand, key_names=names, new_names=new_names)
|
|
292
|
+
return result
|
|
293
|
+
|
|
294
|
+
def visit_GetOp(self, node: GetOp):
|
|
295
|
+
operand = self.visit(node.operand)
|
|
296
|
+
key_names = [node.component]
|
|
297
|
+
result = CLAUSE_OP_MAPPING[GET].validate(operand, key_names)
|
|
298
|
+
return result
|
|
299
|
+
|
|
300
|
+
def visit_PreconditionItem(self, node: PreconditionItem) -> Scalar:
|
|
301
|
+
"""
|
|
302
|
+
Return a ScalarType Boolean with True value is precondition is satisfied otherwise False.
|
|
303
|
+
Example:
|
|
304
|
+
"table_code","ColumnID","RowID","SheetID","column_code","row_code","sheet_code","cell_code","CellID","VariableVID","data_type_code"
|
|
305
|
+
S.01.01.01.01,,,,,,,,,xxxxxxx,BOO
|
|
306
|
+
We can check with table_code or VariableVID, here for now, we use table_code
|
|
307
|
+
"""
|
|
308
|
+
type_ = ScalarFactory().scalar_factory(code="Boolean")
|
|
309
|
+
|
|
310
|
+
return Scalar(type_=type_, name=node.label, origin=node.label)
|
|
311
|
+
|
|
312
|
+
def visit_OperationRef(self, node: OperationRef):
|
|
313
|
+
|
|
314
|
+
operation_code = node.operation_code
|
|
315
|
+
if operation_code not in self.calculations_outputs:
|
|
316
|
+
raise exceptions.SemanticError("1-9", operation_code=operation_code)
|
|
317
|
+
return self.calculations_outputs[operation_code]
|
|
318
|
+
|
|
319
|
+
def visit_WithExpression(self, node: WithExpression):
|
|
320
|
+
return self.visit(node.expression)
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
from typing import List, Union
|
|
2
|
+
|
|
3
|
+
from py_dpm.DataTypes.ScalarTypes import ScalarFactory
|
|
4
|
+
from py_dpm.Exceptions.exceptions import SemanticError
|
|
5
|
+
from py_dpm.Utils.tokens import DPM, STANDARD
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Operand:
|
|
9
|
+
"""
|
|
10
|
+
Superclass of all Symbols.
|
|
11
|
+
|
|
12
|
+
:parameter name: Name of the operand
|
|
13
|
+
:parameter origin: Expression to be used to generate this operand.
|
|
14
|
+
|
|
15
|
+
Example of origin expression:
|
|
16
|
+
|
|
17
|
+
A + B = C -> D = C
|
|
18
|
+
|
|
19
|
+
A: First operand of the addition
|
|
20
|
+
B: Second operand of the addition
|
|
21
|
+
|
|
22
|
+
C: Second operand of the equality
|
|
23
|
+
D: First operand of the equality (Origin: A + B)
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def __init__(self, name: Union[str, None], origin: str):
|
|
27
|
+
self.name = name
|
|
28
|
+
self.origin = origin
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Scalar(Operand):
|
|
32
|
+
"""
|
|
33
|
+
Operand to be used when finding a single Cell Reference or an Item
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
def __init__(self, type_, name, origin):
|
|
37
|
+
super().__init__(name, origin)
|
|
38
|
+
self.type = type_
|
|
39
|
+
|
|
40
|
+
def __repr__(self):
|
|
41
|
+
return "<{class_name}(type='{type}',)>".format(
|
|
42
|
+
class_name=self.__class__.__name__,
|
|
43
|
+
type=self.type
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class Component:
|
|
48
|
+
"""
|
|
49
|
+
Superclass of all components inside a recordset
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
def __init__(self, name: str, type_, parent: str, is_global: bool = False) -> None:
|
|
53
|
+
if type_.__class__ in ScalarFactory().all_types():
|
|
54
|
+
self.name = name
|
|
55
|
+
self.type = type_
|
|
56
|
+
self.parent = parent
|
|
57
|
+
self.is_global = is_global
|
|
58
|
+
else:
|
|
59
|
+
raise Exception("INTERNAL: Wrong data type on Component generation")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class KeyComponent(Component):
|
|
63
|
+
"""
|
|
64
|
+
Component used to specify the row, column, sheet or property in a Recordset
|
|
65
|
+
|
|
66
|
+
:parameter name: Name of the component
|
|
67
|
+
:parameter type_: Data type of the component
|
|
68
|
+
:parameter subtype: Specifies if it is a Standard component (Row, Column or Sheet) or a DPM component
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
def __init__(self, name: str, type_: ScalarFactory().all_types, subtype: str, parent: str, is_global: bool = False) -> None:
|
|
72
|
+
super().__init__(name, type_, parent, is_global)
|
|
73
|
+
self.name = name
|
|
74
|
+
if subtype in (DPM, STANDARD):
|
|
75
|
+
self.subtype = subtype
|
|
76
|
+
else:
|
|
77
|
+
raise SemanticError("2-7", component_name=name)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class FactComponent(Component):
|
|
81
|
+
"""
|
|
82
|
+
Component used to specify the data type of the recordset
|
|
83
|
+
"""
|
|
84
|
+
|
|
85
|
+
def __init__(self, type_: ScalarFactory().all_types, parent: str) -> None:
|
|
86
|
+
super().__init__("f", type_, parent)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class AttributeComponent(Component):
|
|
90
|
+
"""
|
|
91
|
+
Components that are included in the recordset as auxiliar information.
|
|
92
|
+
These components are drop normally except for rename and where operators.
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
def __init__(self, name: str, type_: ScalarFactory().all_types, parent: str) -> None:
|
|
96
|
+
super().__init__(name, type_, parent)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class Structure:
|
|
100
|
+
"""
|
|
101
|
+
Structure for the Recordset. Components must have unique names and only one Fact Component can be present.
|
|
102
|
+
"""
|
|
103
|
+
|
|
104
|
+
def __init__(self, components: List[Component]) -> None:
|
|
105
|
+
components_names = [c.name for c in components if isinstance(c, Component)]
|
|
106
|
+
if len(components_names) != len(set(components_names)) or len(components) != len(components_names):
|
|
107
|
+
raise Exception("Duplicated Component Names, check key_components query.")
|
|
108
|
+
facts_components = [c.name for c in components if isinstance(c, FactComponent)]
|
|
109
|
+
if len(facts_components) > 1:
|
|
110
|
+
raise Exception("Duplicated Fact Component")
|
|
111
|
+
self.components = {
|
|
112
|
+
c.name: c for c in components
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
def get_key_components(self) -> dict:
|
|
116
|
+
dpm_components = self.get_dpm_components()
|
|
117
|
+
standard_components = self.get_standard_components()
|
|
118
|
+
return {**dpm_components, **standard_components}
|
|
119
|
+
|
|
120
|
+
def get_key_components_names(self) -> List[str]:
|
|
121
|
+
return [elto for elto in self.get_key_components()]
|
|
122
|
+
|
|
123
|
+
def get_dpm_components(self) -> dict:
|
|
124
|
+
dpm_components = {
|
|
125
|
+
elto_k: elto_v for elto_k, elto_v in self.components.items() if (isinstance(elto_v, KeyComponent) and elto_v.subtype == DPM)
|
|
126
|
+
}
|
|
127
|
+
return dpm_components
|
|
128
|
+
|
|
129
|
+
def get_standard_components(self):
|
|
130
|
+
standard_components = {
|
|
131
|
+
elto_k: elto_v for elto_k, elto_v in self.components.items() if
|
|
132
|
+
(isinstance(elto_v, KeyComponent) and elto_v.subtype == STANDARD)
|
|
133
|
+
}
|
|
134
|
+
return standard_components
|
|
135
|
+
|
|
136
|
+
def get_standard_components_names(self) -> List[str]:
|
|
137
|
+
return [elto for elto in self.get_standard_components()]
|
|
138
|
+
|
|
139
|
+
def get_fact_component(self):
|
|
140
|
+
fact_component = [
|
|
141
|
+
elto_v for elto_v in self.components.values() if (isinstance(elto_v, FactComponent))
|
|
142
|
+
]
|
|
143
|
+
return fact_component[0]
|
|
144
|
+
|
|
145
|
+
def get_attributes(self):
|
|
146
|
+
attributes = {elto_k: elto_v for elto_k, elto_v in self.components.items() if
|
|
147
|
+
(isinstance(elto_v, AttributeComponent))}
|
|
148
|
+
return attributes
|
|
149
|
+
|
|
150
|
+
def replace_components_parent(self, parent: str):
|
|
151
|
+
for component in self.get_key_components().values():
|
|
152
|
+
component.parent = parent
|
|
153
|
+
|
|
154
|
+
def get_attributes_names(self):
|
|
155
|
+
return [attribute for attribute in self.components if
|
|
156
|
+
isinstance(self.components[attribute], AttributeComponent)]
|
|
157
|
+
|
|
158
|
+
def remove_attributes(self):
|
|
159
|
+
attributes_names = self.get_attributes_names()
|
|
160
|
+
for attribute_name in attributes_names:
|
|
161
|
+
del self.components[attribute_name]
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
class RecordSet(Operand):
|
|
165
|
+
"""
|
|
166
|
+
Recordset are collections of Records that share a same Structure.
|
|
167
|
+
Technically, Recordsets are two-dimensional labelled data structures (tabular),
|
|
168
|
+
which can be assimilated to Relational Tables or Data Frames.
|
|
169
|
+
The columns (fields) of the Recordset are provided by the Components of its Structure.
|
|
170
|
+
The rows of the Recordset are its composing Records.
|
|
171
|
+
|
|
172
|
+
:parameter structure: Structure of the recordset
|
|
173
|
+
:var records: Pandas dataframe to hold the data related to this Recordset
|
|
174
|
+
"""
|
|
175
|
+
|
|
176
|
+
def __init__(self, structure: Structure, name: str, origin: str) -> None:
|
|
177
|
+
if not isinstance(structure, Structure):
|
|
178
|
+
raise Exception("Data Validation Error")
|
|
179
|
+
super().__init__(name, origin)
|
|
180
|
+
self.structure: Structure = structure
|
|
181
|
+
self.records = None
|
|
182
|
+
self.name = name
|
|
183
|
+
self.errors = None
|
|
184
|
+
self.interval = None
|
|
185
|
+
self.default = None
|
|
186
|
+
self.has_only_global_components = all([component.is_global for component in structure.get_key_components().values()])
|
|
187
|
+
|
|
188
|
+
def get_key_components(self) -> dict:
|
|
189
|
+
return self.structure.get_key_components()
|
|
190
|
+
|
|
191
|
+
def get_key_components_names(self) -> List[str]:
|
|
192
|
+
return self.structure.get_key_components_names()
|
|
193
|
+
|
|
194
|
+
def get_dpm_components(self):
|
|
195
|
+
return self.structure.get_dpm_components()
|
|
196
|
+
|
|
197
|
+
def get_standard_components(self):
|
|
198
|
+
return self.structure.get_standard_components()
|
|
199
|
+
|
|
200
|
+
def get_standard_components_names(self) -> List[str]:
|
|
201
|
+
return self.structure.get_standard_components_names()
|
|
202
|
+
|
|
203
|
+
def get_fact_component(self):
|
|
204
|
+
return self.structure.get_fact_component()
|
|
205
|
+
|
|
206
|
+
def get_attributes(self):
|
|
207
|
+
return self.structure.get_attributes()
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
class ScalarSet(Operand):
|
|
211
|
+
"""
|
|
212
|
+
Scalar set are a collection of scalars used in the IN operator.
|
|
213
|
+
"""
|
|
214
|
+
|
|
215
|
+
def __init__(self, type_, name, origin):
|
|
216
|
+
super().__init__(name, origin)
|
|
217
|
+
self.type = type_
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
class ConstantOperand(Scalar):
|
|
221
|
+
def __init__(self, type_, name, origin, value):
|
|
222
|
+
super().__init__(name=name, origin=origin, type_=type_)
|
|
223
|
+
self.value = value # TODO: Check this
|