pydpm_xl 0.2.2__py3-none-any.whl → 0.2.3__py3-none-any.whl

py_dpm/dpm_xl/validation/utils.py

@@ -1,98 +0,0 @@
-import numpy as np
-import pandas as pd
-
-from py_dpm.dpm_xl.validation.generation_utils import ValidationsGenerationUtils
-from py_dpm.dpm_xl.utils.tokens import *
-
-
-def generate_context_structure(lst: list, group_df: pd.DataFrame):
-    """
-    Method to generate the structure of the contexts of variables in order to generate the hierarchy validations
-    :param lst: list to store the structure of contexts
-    :param group_df: Dataframe with information about properties and items of variables
-    """
-    group_df.set_index(CONTEXT_PROPERTY, inplace=True)
-    result = group_df.to_dict()[CONTEXT_ITEM]
-    result[VARIABLE_VID] = group_df[VARIABLE_VID].tolist()[0]
-    result[VARIABLE_PROPERTY_ID] = group_df[VARIABLE_PROPERTY_ID].tolist()[0]
-
-    lst.append(result)
-
-
-def generate_subexpression(common_values, data, table, operator:str=None):
-    """
-    Method to generate the subexpression of a hierarchy expression
-    :param common_values: dictionary with information about the common components and their values
-    :param data: variable datapoints
-    :param table: table code
-    """
-    abs_value = True if operator and operator != '=' else False
-    if ROW_CODE in common_values:
-        rows = None
-    else:
-        rows = ROW + data[ROW_CODE].iloc[0] if data[ROW_CODE].iloc[0] else None
-
-    if COLUMN_CODE in common_values:
-        cols = None
-    else:
-        cols = COLUMN + data[COLUMN_CODE].iloc[0] if data[COLUMN_CODE].iloc[0] else None
-
-    if SHEET_CODE in common_values:
-        sheets = None
-    else:
-        sheets = SHEET + data[SHEET_CODE].iloc[0] if data[SHEET_CODE].iloc[0] else None
-    
-    if abs_value:
-        expr = ValidationsGenerationUtils.write_cell(table, rows, cols, sheets)
-        return 'abs(' + expr + ')'
-    
-    return ValidationsGenerationUtils.write_cell(table, rows, cols, sheets)
-
-
-def get_common_components(left_data: pd.DataFrame, right_data: pd.DataFrame):
-    """
-    Method to get the common components of expression operands
-    :param left_data: Dataframe corresponding to the data of the left operand
-    :param right_data: Dataframe corresponding to the data of the right side operands
-    """
-    common_components = {}
-    for component in CELL_COMPONENTS:
-        left = left_data[component].dropna().tolist()
-        right = right_data[component].isnull().values.any()
-        if left and not right:
-            intersection_component = np.intersect1d(left_data[component], right_data[component])
-            if len(intersection_component):
-                if all(right_data[component].isin(intersection_component)):
-                    common_components[component] = intersection_component
-    return common_components
-
-
-def group_hierarchy_right_data(right_data: pd.DataFrame, left_table: str, common_components: list, comparisson_operator:str=None):
-    """
-    Method to group right side operands
-    :param right_data: Dataframe with the datapoints of the right side of expression
-    :param left_table: Table code of left operand
-    :param common_components: List with the common components of right and left operands
-    """
-    right_table = right_data[TABLE_CODE].tolist()[0]
-    is_table_unique = left_table == right_table
-    right_operands_lst = []
-    right_table_cell = right_table if not is_table_unique else None
-    # Here we have to add the order of the items in the expression
-    order_dict = {}
-    for keys, group in right_data.groupby([ARITHMETIC_OPERATOR_SYMBOL, ITEM_ID]):
-        operand = generate_subexpression(common_components, group, right_table_cell,comparisson_operator)
-        operand = keys[0] + operand
-        if ORDER in group:
-            order_dict[group[ORDER].tolist()[0]] = operand
-        else:
-            order_dict[group[ORDER + '_x'].tolist()[0]] = operand
-
-    right_operands_lst = [order_dict[order] for order in sorted(order_dict)]
-
-    right_expression = ' '.join(right_operands_lst)
-    if DUPLICATE_VARIABLES not in right_data:
-        right_data[DUPLICATE_VARIABLES] = False
-    duplicate_variables = right_data[DUPLICATE_VARIABLES].any()
-    right_items = right_data[ITEM_ID].unique().tolist()
-    return [right_expression, right_table, duplicate_variables,right_items]

py_dpm/dpm_xl/validation/variants.py

@@ -1,359 +0,0 @@
-import copy
-import re
-
-import pandas as pd
-
-from py_dpm.dpm_xl.ast.nodes import VarID, WithExpression
-from py_dpm.dpm_xl.ast.template import ASTTemplate
-from py_dpm.exceptions import exceptions
-from py_dpm.dpm.models import (
-    ModuleVersionComposition,
-    TableGroup,
-    TableGroupComposition,
-    TableVersion,
-    ViewDatapoints,
-)
-from py_dpm.dpm_xl.utils.tokens import (
-    EXPRESSION,
-    STATUS,
-    STATUS_CORRECT,
-    STATUS_INCOMPLETE,
-    STATUS_INCORRECT,
-    VALIDATION_CODE,
-)
-from py_dpm.dpm_xl.utils.data_handlers import filter_all_data
-
-cell_components = ["table", "rows", "cols", "sheets"]
-TABLE_ID = "TableID"
-MODULE_VID = "ModuleVID"
-TABLE_CODE = "Code"
-GROUP = "group"
-MODULE_VERSION_ID = "module_version_id"
-MODULE_CODE = "module_code"
-
-
-class VariantsProcessorChecker(ASTTemplate):
-    def __init__(self, ast):
-        super().__init__()
-        self._is_variant = False
-        self.visit(ast)
-
-    @property
-    def is_variant(self):
-        return self._is_variant
-
-    def visit_VarID(self, node: VarID):
-        if node.is_table_group:
-            self._is_variant = True
-
-
-class VariantsProcessor(ASTTemplate):
-    """
-    Class to generate individual validations from a validation defined on groups.
-
-    :parameter expression: DPM-XL expression.
-    :parameter ast: Abstract Syntax Tree of expression.
-    :parameter session: SQLAlchemy Session to be used to connect to the DB.
-    :parameter validation_code: Code of parent validation.
-    """
-
-    def __init__(self, expression, ast, session, validation_code, release_id):
-        super().__init__()
-        self.expression = expression
-        self.AST = ast
-        self.session = session
-        self.validation_code = validation_code
-        self.current_validation = 1
-        self.partial_selection = None
-        self.release_id = release_id
-
-        self.table_data = {}
-        self.group_tables = []
-        self.table_groups_compositions = {}
-        self.tables_without_cells = []
-        self.children_suffix = None
-        self.equal_children_suffix = True
-
-        self.visit(self.AST)
-
-    def check_if_table_has_cells(self, table_code, rows, cols, sheets):
-        """
-        Method to check if an individual table has cells for rows, columns and sheets of validation.
-        :param table_code: code of table.
-        :param rows: rows.
-        :param cols: columns.
-        :param sheets: sheets.
-        :return: True if table has cells, False otherwise.
-        """
-        if table_code not in self.table_data:
-            datapoints = ViewDatapoints.get_table_data(
-                session=self.session, table=table_code, release_id=self.release_id
-            )
-            self.table_data[table_code] = datapoints
-        else:
-            datapoints = self.table_data[table_code]
-        data = filter_all_data(
-            data=datapoints, table_code=table_code, rows=rows, cols=cols, sheets=sheets
-        )
-        return not data.empty
-
-    def check_table_from_module(self, node, table_module_id, is_abstract):
-        """
-        Method to get table_versions of a table group when there are cells for those tables.
-        :param node: var_id node
-        :param table_module_id: module id
-        :param is_abstract: flag that represents if table is abstract
-        :return: table version if table version has cells, None otherwise.
-        """
-
-        table_versions = TableVersion.get_tables_versions_of_table_group_compositions(
-            session=self.session,
-            table_id=table_module_id,
-            is_abstract=is_abstract,
-            release_id=self.release_id,
-        )
-
-        for table_version in table_versions:
-            table_with_cells = self.check_if_table_has_cells(
-                table_version.Code, node.rows, node.cols, node.sheets
-            )
-            if table_with_cells:
-                return table_version
-            else:
-                if table_version.TableID not in self.tables_without_cells:
-                    self.tables_without_cells.append(table_version.TableID)
-        return None
-
-    def generate_child_expression(self, group_code, table_code):
-        """
-        Method to replace group_code by table_code in order to generate new validations.
-        :param group_code: Group code.
-        :param table_code: Table code.
-        """
-        groups = re.search("(" + re.escape(group_code) + "[^.$]" + ")", self.expression)
-        if f"g{group_code}" in self.expression and groups:
-            group = groups.group(0)
-            suffix = table_code[-(len(table_code) - len(group_code)) :]
-            if group:
-                if not self.children_suffix:
-                    self.children_suffix = suffix
-                else:
-                    if self.children_suffix != suffix:
-                        self.equal_children_suffix = False
-                        return
-                self.expression = re.sub(
-                    re.escape(f"g{group_code}") + "[^.$]",
-                    f"t{table_code}" + group[-1],
-                    self.expression,
-                )
-
-    def generate_child_expressions(self):
-        """
-        Method to generate all the individual validations.
-        :return: Expressions and expressions with errors.
-        """
-        if self.group_tables:
-            final_expressions = []
-            final_expressions_with_errors = []
-            df_tables = pd.DataFrame.from_records(self.group_tables)
-            df_tables = df_tables[~df_tables[TABLE_ID].isin(self.tables_without_cells)]
-            df_tables.drop_duplicates(inplace=True)
-
-            table_ids = df_tables[TABLE_ID].tolist()
-            modules_df = ModuleVersionComposition.get_modules_from_table_ids(
-                session=self.session, table_ids=table_ids, release_id=self.release_id
-            )
-
-            data = pd.merge(df_tables, modules_df, on=[TABLE_ID])
-
-            expression = copy.deepcopy(self.expression)
-            for module, modules_tables in data.groupby(MODULE_VID):
-                modules_tables.apply(
-                    lambda x: self.generate_child_expression(x[GROUP], x[TABLE_CODE]),
-                    axis=1,
-                )
-                module_code = modules_tables[MODULE_CODE].unique().tolist()[0]
-                if len(modules_tables) < len(self.table_groups_compositions):
-                    final_expressions_with_errors.append(
-                        {
-                            EXPRESSION: self.expression,
-                            MODULE_VERSION_ID: module,
-                            MODULE_CODE: module_code,
-                        }
-                    )
-                else:
-                    if self.equal_children_suffix:
-                        final_expressions.append(
-                            {
-                                EXPRESSION: self.expression,
-                                MODULE_VERSION_ID: module,
-                                MODULE_CODE: module_code,
-                            }
-                        )
-                    else:
-                        final_expressions_with_errors.append(
-                            {
-                                EXPRESSION: self.expression,
-                                MODULE_VERSION_ID: module,
-                                MODULE_CODE: module_code,
-                            }
-                        )
-                        self.equal_children_suffix = True
-                self.expression = copy.deepcopy(expression)
-                self.children_suffix = None
-
-            return final_expressions, final_expressions_with_errors
-        else:
-            raise exceptions.SemanticError("5-2-1")
-
-    def create_validation(self, expression_info, status):
-        """
-        Method to centralize creation of validations given expression and status of new validation
-        :param expression_info: Dictionary with information about expression, module_vid and module_code of validation
-        :param status: Status of validation
-        :return: Validation with code, expression and status
-        """
-
-        validation_code = (
-            self.generate_child_validation_code() if status == STATUS_CORRECT else None
-        )
-
-        validation = {
-            VALIDATION_CODE: validation_code,
-            EXPRESSION: expression_info[EXPRESSION],
-            STATUS: status,
-            MODULE_VERSION_ID: expression_info[MODULE_VERSION_ID],
-            MODULE_CODE: expression_info[MODULE_CODE],
-        }
-        return validation
-
-    def create_validation_new_format(self, expressions_dict):
-        """
-        Method to centralize creation of validations given expression and status of new validation
-        :param expression_info: Dictionary with information about expression, module_vid and module_code of validation
-        :param status: Status of validation
-        :return: Validation with code, expression and status
-        """
-
-        correct_expressions = expressions_dict[STATUS_CORRECT]
-        incomplete_expressions = expressions_dict[STATUS_INCOMPLETE]
-        incorrect_expressions = expressions_dict[STATUS_INCORRECT]
-        unique_correct_expressions = (
-            pd.DataFrame.from_records(correct_expressions)["expression"]
-            .unique()
-            .tolist()
-            if correct_expressions
-            else []
-        )
-        unique_incomplete_expressions = (
-            pd.DataFrame.from_records(incomplete_expressions)["expression"]
-            .unique()
-            .tolist()
-            if incomplete_expressions
-            else []
-        )
-        unique_incorrect_expressions = (
-            pd.DataFrame.from_records(incorrect_expressions)["expression"]
-            .unique()
-            .tolist()
-            if incorrect_expressions
-            else []
-        )
-        validations = []
-        if correct_expressions:
-            v = self._aux_create_validation_new_format(
-                correct_expressions, unique_correct_expressions, STATUS_CORRECT
-            )
-            validations.extend(v)
-        if incomplete_expressions:
-            v = self._aux_create_validation_new_format(
-                incomplete_expressions, unique_incomplete_expressions, STATUS_INCOMPLETE
-            )
-            validations.extend(v)
-        if incorrect_expressions:
-            v = self._aux_create_validation_new_format(
-                incorrect_expressions, unique_incorrect_expressions, STATUS_INCORRECT
-            )
-            validations.extend(v)
-
-        return validations
-
-    def _aux_create_validation_new_format(
-        self, expressions, unique_expressions, status
-    ):
-        validations = []
-        for expr in unique_expressions:
-            aux = {}
-            aux[EXPRESSION] = expr
-            aux[STATUS] = status
-            if status == STATUS_CORRECT:
-                aux[VALIDATION_CODE] = self.generate_child_validation_code()
-            else:
-                aux[VALIDATION_CODE] = None
-            aux["scopes"] = []
-            for elto in expressions:
-                if elto[EXPRESSION] == aux[EXPRESSION]:
-                    aux["scopes"].append(
-                        {
-                            "module_versions_ids": [elto[MODULE_VERSION_ID]],
-                            "module_code": elto[MODULE_CODE],
-                        }
-                    )
-            validations.append(aux)
-        return validations
-
-    def generate_child_validation_code(self):
-        """
-        Method to calculate validation codes for new validations.
-        :return: New code for the individual validation.
-        """
-        child_code = f"{self.validation_code}-{self.current_validation}"
-        self.current_validation += 1
-        return child_code
-
-    def visit_WithExpression(self, node: WithExpression):
-        self.partial_selection = node.partial_selection
-        self.visit(node.partial_selection)
-        self.visit(node.expression)
-
-    def visit_VarID(self, node: VarID):
-        if self.partial_selection:
-            for attribute in cell_components:
-                if not getattr(node, attribute, False) and hasattr(
-                    self.partial_selection, attribute
-                ):
-                    setattr(node, attribute, getattr(self.partial_selection, attribute))
-
-        if node.is_table_group or self.partial_selection.is_table_group:
-            if node.table:
-                if node.table not in self.table_groups_compositions:
-                    group: TableGroup = TableGroup.get_group_from_code(
-                        session=self.session, group_code=node.table
-                    )
-                    if not group:
-                        raise exceptions.SemanticError("1-6", table_group=node.table)
-                    table_groups_compositions = (
-                        TableGroupComposition.get_from_parent_table_code(
-                            code=node.table, session=self.session
-                        )
-                    )
-                    self.table_groups_compositions[node.table] = (
-                        table_groups_compositions
-                    )
-                else:
-                    table_groups_compositions = self.table_groups_compositions[
-                        node.table
-                    ]
-
-                for table_id, is_abstract in table_groups_compositions:
-                    table_version = self.check_table_from_module(
-                        node, table_id, is_abstract
-                    )
-                    if table_version:
-                        self.group_tables.append(
-                            {
-                                GROUP: node.table,
-                                TABLE_CODE: table_version.Code,
-                                TABLE_ID: table_version.TableID,
-                            }
-                        )