pydpm_xl 0.1.10__py3-none-any.whl

py_dpm/AST/ModuleDependencies.py

@@ -0,0 +1,203 @@
+from abc import ABC
+
+from py_dpm.AST.ASTObjects import *
+from py_dpm.AST.ASTTemplate import ASTTemplate
+from py_dpm.AST.WhereClauseChecker import WhereClauseChecker
+from py_dpm.Exceptions import exceptions
+from py_dpm.models import TableVersion, VariableVersion, ViewDatapoints, ViewModules, ItemCategory
+
+operand_elements = ['table', 'rows', 'cols', 'sheets', 'default', 'interval']
+
+def filter_datapoints_df(df, table, table_info:dict, release_id:int=None):
+    """
+    """
+    mapping_dictionary = {'rows': 'row_code', 'cols': 'column_code', 'sheets': 'sheet_code'}
+    df = df[df['table_code'] == table]
+    for k, v in table_info.items():
+        if v is not None:
+            if '-' in v[0]:
+                low_limit, high_limit = v[0].split('-')
+                df = df[(df[mapping_dictionary[k]].between(low_limit, high_limit))]
+            elif v[0] == '*':
+                continue
+            else:
+                df = df[(df[mapping_dictionary[k]].isin(v))]
+
+    if release_id:
+        df = df[df['release_id'] == release_id]
+    return df
+
+def filter_module_by_table_df(df, table):
+    """
+    Returns a list of modules that contain the table
+    """
+    module_list = df[df['table_code'] == table]['module_code'].tolist()
+    return module_list
+
+class ModuleDependencies(ASTTemplate, ABC):
+    def __init__(self, session, ast, release_id, date, module_ref, is_scripting=False):
+        self.release_id = release_id
+        self.AST = ast
+        self.tables = {}
+        self.operands = {}
+        self.full_operands = {}
+        self.module_ref = module_ref
+        # self.key_components = {}
+        self.partial_selection = None
+        self.data = None
+        self.items = []
+        self.preconditions = False
+        self.dimension_codes = []
+        # self.open_keys = None
+
+        self.operations = []
+        self.operations_data = None
+        self.is_scripting = is_scripting
+
+        self.session = session
+        self.time_period = 't'  # TODO
+        self.date = date
+        self.modules = {}
+        self.from_time_shift = False
+
+        super().__init__()
+        self.visit(self.AST)
+
+
+    # Start of visiting nodes
+    def visit_WithExpression(self, node: WithExpression):
+        if node.partial_selection.is_table_group:
+            raise exceptions.SemanticError("1-10", table=node.partial_selection.table)
+        self.partial_selection: VarID = node.partial_selection
+        self.visit(node.expression)
+
+    def visit_VarID(self, node: VarID):
+
+        if node.is_table_group:
+            raise exceptions.SemanticError("1-10", table=node.table)
+
+        if self.partial_selection:
+            for attribute in operand_elements:
+                if getattr(node, attribute, None) is None and not getattr(self.partial_selection, attribute, None) is None:
+                    setattr(node, attribute, getattr(self.partial_selection, attribute))
+
+        if not node.table:
+            raise exceptions.SemanticError("1-4", table=node.table)
+
+        table_info = {
+            'rows': node.rows,
+            'cols': node.cols,
+            'sheets': node.sheets
+        }
+
+        if node.table not in self.tables:
+            self.tables[node.table] = table_info
+            self.operands[node.table] = [self.time_period]
+            # self.operands[node.table] = [node]
+        else:
+            if self.time_period not in self.operands[node.table]:
+                self.operands[node.table].append(self.time_period)
+                # self.operands[node.table].append(node)
+
+        # Variables full
+        variables_full = ViewDatapoints.get_filtered_datapoints(self.session,
+                                                                node.table,
+                                                                table_info,
+                                                                release_id=self.release_id)
+        # variables_full = ViewDatapoints.get_table_data(self.session, node.table, table_info['rows'], table_info['cols'], table_info['sheets'],
+        #                                              self.release_id)
+
+        if variables_full.empty:
+            raise exceptions.SemanticError("1-5", open_keys=table_info)
+
+        final_list = variables_full['variable_id'].to_list()
+        # Here change table for module or modules
+        modules = ViewModules().get_all_modules(self.session)  # TODO
+        modules = filter_module_by_table_df(modules, node.table)
+        # modules = ViewModules().get_modules(self.session, [node.table], None)
+        if self.module_ref:
+            if self.module_ref in modules:
+                full_name = f"{self.module_ref}:{self.time_period}"
+                if full_name not in self.full_operands:
+                    self.full_operands[full_name] = final_list
+                else:
+                    for elto in final_list:
+                        if elto not in self.full_operands[full_name]:
+                            self.full_operands[full_name].append(elto)
+            else:
+                for module in modules:
+                    full_name = f"{module}:{self.time_period}"
+                    if full_name not in self.full_operands:
+                        self.full_operands[full_name] = final_list
+                    else:
+                        for elto in final_list:
+                            if elto not in self.full_operands[full_name]:
+                                self.full_operands[full_name].append(elto)
+        else:
+            full_name = f"{node.table}:{self.time_period}"
+            self.full_operands[full_name] = final_list
+
+
+    def visit_Dimension(self, node: Dimension):
+        if node.dimension_code not in self.dimension_codes:
+            self.dimension_codes.append(node.dimension_code)
+            if not ItemCategory.get_property_from_code(node.dimension_code, self.session):
+                raise exceptions.SemanticError("1-5", open_keys=node.dimension_code)
+
+    def visit_VarRef(self, node: VarRef):
+        if not VariableVersion.check_variable_exists(self.session, node.variable, self.release_id):
+            raise exceptions.SemanticError('1-3', variable=node.variable)
+
+    def visit_PreconditionItem(self, node: PreconditionItem):
+
+        if self.is_scripting:
+            raise exceptions.SemanticError('6-3', precondition=node.variable_id)
+
+        if not TableVersion.check_table_exists(self.session, node.variable_id, self.release_id):
+            raise exceptions.SemanticError("1-3", variable=node.variable_id)
+
+        self.preconditions = True
+
+    def visit_Scalar(self, node: Scalar):
+        if node.item and node.scalar_type == 'Item':
+            if node.item not in self.items:
+                self.items.append(node.item)
+                if not ItemCategory.get_item_category_id_from_signature(node.item, self.session):
+                    raise exceptions.SemanticError("1-1", items=node.item)
+
+
+    def visit_WhereClauseOp(self, node: WhereClauseOp):
+        self.visit(node.operand)
+        checker = WhereClauseChecker()
+        checker.visit(node.condition)
+        node.key_components = checker.key_components
+        self.visit(node.condition)
+
+    def visit_TimeShiftOp(self, node: TimeShiftOp):
+        self.from_time_shift = True
+        period_indicator = node.period_indicator
+        shift_number = node.shift_number
+        # compute new time period
+        if period_indicator not in ('A', 'Q', 'M', 'W', 'D'):
+            raise ValueError("Period indicator is not valid")
+        if "-" in shift_number:
+            new_time_period = f"t+{period_indicator}{shift_number}"
+        else:
+            new_time_period = f"t-{period_indicator}{shift_number}"
+
+        self.time_period = new_time_period
+        self.visit(node.operand)
+        self.from_time_shift = False
+
+    def visit_OperationRef(self, node: OperationRef):
+        if not self.is_scripting:
+            raise exceptions.SemanticError("6-2", operation_code=node.operation_code)
+
+    def visit_PersistentAssignment(self, node: PersistentAssignment):
+        # TODO: visit node.left when there are calculations variables in database
+        self.visit(node.right)
+
+    def visit_TemporaryAssignment(self, node: TemporaryAssignment):
+        temporary_identifier = node.left
+        self.operations.append(temporary_identifier.value)
+        self.visit(node.right)

py_dpm/AST/WhereClauseChecker.py

@@ -0,0 +1,12 @@
+from py_dpm.AST.ASTObjects import Dimension
+from py_dpm.AST.ASTTemplate import ASTTemplate
+
+
+class WhereClauseChecker(ASTTemplate):
+
+    def __init__(self):
+        super().__init__()
+        self.key_components = []
+
+    def visit_Dimension(self, node: Dimension):
+        self.key_components.append(node.dimension_code)

py_dpm/AST/check_operands.py

@@ -0,0 +1,302 @@
+from abc import ABC
+
+import pandas as pd
+
+from py_dpm.AST.ASTObjects import Dimension, OperationRef, PersistentAssignment, PreconditionItem, \
+    Scalar, TemporaryAssignment, VarID, VarRef, WhereClauseOp, WithExpression
+from py_dpm.AST.ASTTemplate import ASTTemplate
+from py_dpm.AST.WhereClauseChecker import WhereClauseChecker
+from py_dpm.DataTypes.ScalarTypes import Integer, Mixed, Number, ScalarFactory
+from py_dpm.Exceptions import exceptions
+from py_dpm.Exceptions.exceptions import SemanticError
+from py_dpm.models import ItemCategory, Operation, VariableVersion, ViewDatapoints, \
+    ViewKeyComponents, ViewOpenKeys
+from py_dpm.Utils.operands_mapping import generate_new_label, set_operand_label
+from py_dpm.data_handlers import filter_all_data
+
+operand_elements = ['table', 'rows', 'cols', 'sheets', 'default', 'interval']
+
+
+def _create_operand_label(node):
+    label = generate_new_label()
+    node.label = label
+
+
+def _modify_element_info(new_data, element, table_info):
+    if new_data is None and table_info[element] is None:
+        pass
+    elif table_info[element] == ['*']:
+        pass
+    elif new_data is not None and table_info[element] is None:
+        table_info[element] = new_data
+
+    elif new_data == ['*']:
+        # We have already all data available
+        table_info[element] = new_data
+
+    else:
+        # We get only the elements that are not already in the info and sort them
+        new_list = [x for x in new_data if x not in table_info[element]]
+        new_list += table_info[element]
+        new_list = sorted(new_list)
+        table_info[element] = new_list
+
+
+def _modify_table(node, table_info):
+    for element in table_info:
+        _modify_element_info(getattr(node, element), element, table_info)
+
+
+def format_missing_data(node):
+    rows = ', '.join([f"r{x}" for x in node.rows]) if node.rows else None
+    cols = ', '.join([f"c{x}" for x in node.cols]) if node.cols else None
+    sheets = ', '.join([f"s{x}" for x in node.sheets]) if node.sheets else None
+    op_pos = [node.table, rows, cols, sheets]
+    cell_exp = ", ".join(x for x in op_pos if x is not None)
+    raise exceptions.SemanticError("1-2", cell_expression=cell_exp)
+
+
+class OperandsChecking(ASTTemplate, ABC):
+    def __init__(self, session, expression, ast, release_id, is_scripting=False):
+        self.expression = expression
+        self.release_id = release_id
+        self.AST = ast
+        self.tables = {}
+        self.operands = {}
+        self.key_components = {}
+        self.partial_selection = None
+        self.data = None
+        self.items = []
+        self.preconditions = False
+        self.dimension_codes = []
+        self.open_keys = None
+
+        self.operations = []
+        self.operations_data = None
+        self.is_scripting = is_scripting
+
+        self.session = session
+
+        super().__init__()
+        self.visit(self.AST)
+        self.check_headers()
+        self.check_items()
+        self.check_tables()
+        self.check_dimensions()
+
+        self.check_operations()
+
+    def _check_header_present(self, table, header):
+        if (self.partial_selection is not None and self.partial_selection.table == table and
+                getattr(self.partial_selection, header) is not None):
+            return
+        for node in self.operands[table]:
+            if getattr(node, header) is None:
+                if header == 'cols':
+                    header = 'columns'
+                raise exceptions.SemanticError("1-20", header=header, table=table)
+
+    def check_headers(self):
+        table_codes = list(self.tables.keys())
+        if len(table_codes) == 0:
+            return
+        query = """
+        SELECT DISTINCT tv.Code, tv.StartReleaseID, tv.EndReleaseID, h.Direction, t.HasOpenRows, t.HasOpenColumns, t.HasOpenSheets
+        FROM [Table] AS t
+        INNER JOIN TableVersion tv ON t.TableID = tv.TableID
+        INNER JOIN TableVersionHeader tvh ON tv.TableVID = tvh.TableVID
+        INNER JOIN Header h ON h.HeaderID = tvh.HeaderID
+        """
+        codes = [f"{code!r}" for code in table_codes]
+        query += f"WHERE tv.Code IN ({', '.join(codes)})"
+        query += "AND tv.EndReleaseID is null"
+        df_headers = pd.read_sql(query, self.session.connection())
+        for table in table_codes:
+            table_headers = df_headers[df_headers['Code'] == table]
+            open_rows = table_headers['HasOpenRows'].values[0]
+            open_cols = table_headers['HasOpenColumns'].values[0]
+            open_sheets = table_headers['HasOpenSheets'].values[0]
+            if table_headers.empty:
+                continue
+            if "Y" in table_headers['Direction'].values and not open_rows:
+                self._check_header_present(table, 'rows')
+            if "X" in table_headers['Direction'].values and not open_cols:
+                self._check_header_present(table, 'cols')
+            if "Z" in table_headers['Direction'].values and not open_sheets:
+                self._check_header_present(table, 'sheets')
+
+
+    def check_items(self):
+        if len(self.items) == 0:
+            return
+        df_items = ItemCategory.get_items(self.session, self.items, self.release_id)
+        if len(df_items.iloc[:, 0].values) < len(set(self.items)):
+            not_found_items = list(set(self.items).difference(set(df_items['Signature'])))
+            raise exceptions.SemanticError("1-1", items=not_found_items)
+
+    def check_dimensions(self):
+        if len(self.dimension_codes) == 0:
+            return
+        self.open_keys = ViewOpenKeys.get_keys(self.session, self.dimension_codes, self.release_id)
+        if len(self.open_keys) < len(self.dimension_codes):
+            not_found_dimensions = list(set(self.dimension_codes).difference(self.open_keys['property_code']))
+            raise exceptions.SemanticError("1-5", open_keys=not_found_dimensions)
+
+    def check_tables(self):
+        for table, value in self.tables.items():
+            # Extract all data and filter to get only necessary data
+            table_info = value
+            df_table = ViewDatapoints.get_table_data(self.session, table, table_info['rows'],
+                                                table_info['cols'], table_info['sheets'], self.release_id)
+            if df_table.empty:
+                cell_expression = f'table: {table}'
+                for k, v in table_info.items():
+                    if v:
+                        cell_expression += f' {k}: {v}'
+                raise exceptions.SemanticError("1-2", cell_expression=cell_expression)
+
+            # Insert data type on each node by selecting only data required by node
+            for node in self.operands[table]:
+                node_data = filter_all_data(df_table, table, node.rows, node.cols, node.sheets)
+                # Checking grey cells (no variable ID in data for that cell)
+                grey_cells_data = node_data[node_data['variable_id'].isnull()]
+                if not grey_cells_data.empty:
+                    if len(grey_cells_data) > 10:
+                        list_cells = grey_cells_data["cell_code"].values[:10]
+                    else:
+                        list_cells = grey_cells_data["cell_code"].values
+                    cell_expression = ', '.join(list_cells)
+                    raise exceptions.SemanticError("1-17", cell_expression=cell_expression)
+                if node_data.empty:
+                    format_missing_data(node)
+                extract_data_types(node, node_data['data_type'])
+                if df_table['sheet_code'].isnull().all() and node.sheets is not None and '*' in node.sheets:
+                    raise SemanticError("1-18")
+                if df_table['row_code'].isnull().all() and node.rows is not None and '*' in node.rows:
+                    raise SemanticError("1-19")
+            del node_data
+
+            # Adding data to self.data
+            if self.data is None:
+                self.data = df_table
+            else:
+                self.data: pd.DataFrame = pd.concat([self.data, df_table], axis=0).reset_index(drop=True)
+
+
+            self.key_components[table] = ViewKeyComponents.get_by_table(self.session, table, self.release_id)
+
+    # Start of visiting nodes
+    def visit_WithExpression(self, node: WithExpression):
+        if node.partial_selection.is_table_group:
+            raise exceptions.SemanticError("1-10", table=node.partial_selection.table)
+        self.partial_selection: VarID = node.partial_selection
+        self.visit(node.expression)
+
+    def visit_VarID(self, node: VarID):
+
+        if node.is_table_group:
+            raise exceptions.SemanticError("1-10", table=node.table)
+
+        if self.partial_selection:
+            for attribute in operand_elements:
+                if getattr(node, attribute, None) is None and not getattr(self.partial_selection, attribute, None) is None:
+                    setattr(node, attribute, getattr(self.partial_selection, attribute))
+
+        if not node.table:
+            raise exceptions.SemanticError("1-4", table=node.table)
+
+        _create_operand_label(node)
+        set_operand_label(node.label, node)
+
+        table_info = {
+            'rows': node.rows,
+            'cols': node.cols,
+            'sheets': node.sheets
+        }
+
+        if node.table not in self.tables:
+            self.tables[node.table] = table_info
+            self.operands[node.table] = [node]
+        else:
+            self.operands[node.table].append(node)
+            _modify_table(node, self.tables[node.table])
+
+    def visit_Dimension(self, node: Dimension):
+        if node.dimension_code not in self.dimension_codes:
+            self.dimension_codes.append(node.dimension_code)
+
+    def visit_VarRef(self, node: VarRef):
+        if not VariableVersion.check_variable_exists(self.session, node.variable, self.release_id):
+            raise exceptions.SemanticError('1-3', variable=node.variable)
+
+    def visit_PreconditionItem(self, node: PreconditionItem):
+
+        if self.is_scripting:
+            raise exceptions.SemanticError('6-3', precondition=node.variable_code)
+
+        if not VariableVersion.check_variable_exists(self.session, node.variable_code, self.release_id):
+            raise exceptions.SemanticError("1-3", variable=node.variable_code)
+
+        self.preconditions = True
+        _create_operand_label(node)
+        set_operand_label(node.label, node)
+
+    def visit_Scalar(self, node: Scalar):
+        if node.item and node.scalar_type == 'Item':
+            if node.item not in self.items:
+                self.items.append(node.item)
+
+    def visit_WhereClauseOp(self, node: WhereClauseOp):
+        self.visit(node.operand)
+        checker = WhereClauseChecker()
+        checker.visit(node.condition)
+        node.key_components = checker.key_components
+        self.visit(node.condition)
+
+    def visit_OperationRef(self, node: OperationRef):
+        if not self.is_scripting:
+            raise exceptions.SemanticError("6-2", operation_code=node.operation_code)
+
+    def visit_PersistentAssignment(self, node: PersistentAssignment):
+        # TODO: visit node.left when there are calculations variables in database
+        self.visit(node.right)
+
+    def visit_TemporaryAssignment(self, node: TemporaryAssignment):
+        temporary_identifier = node.left
+        self.operations.append(temporary_identifier.value)
+        self.visit(node.right)
+
+    def check_operations(self):
+        if len(self.operations):
+            df_operations = Operation.get_operations_from_codes(session=self.session, operation_codes=self.operations,
+                                                                release_id=self.release_id)
+            if len(df_operations.values) < len(self.operations):
+                not_found_operations = list(set(self.operations).difference(set(df_operations['Code'])))
+                raise exceptions.SemanticError('1-8', operations=not_found_operations)
+            self.operations_data = df_operations
+
+
+def extract_data_types(node: VarID, database_types: pd.Series) -> None:
+    """
+    Extract data type of var ids from database information
+    :param node: Var id
+    :param database_types: Series that contains the data types of node elements
+    :return: None
+    """
+    unique_types = database_types.unique()
+    scalar_factory = ScalarFactory()
+    if len(unique_types) == 1:
+        data_type = scalar_factory.database_types_mapping(unique_types[0])
+        if node.interval and isinstance(data_type(), Number):
+            setattr(node, "type", data_type(node.interval))
+        else:
+            setattr(node, "type", data_type())
+    else:
+        data_types = {scalar_factory.database_types_mapping(data_type) for data_type in unique_types}
+        if len(data_types) == 1:
+            data_type = data_types.pop()
+            setattr(node, "type", data_type())
+        elif len(data_types) == 2 and Number in data_types and Integer in data_types:
+            setattr(node, "type", Number())
+        else:
+            setattr(node, "type", Mixed())