pydpm_xl 0.1.10__py3-none-any.whl

py_dpm/ValidationsGeneration/Utils.py

@@ -0,0 +1,364 @@
+import json
+import warnings
+from datetime import datetime
+from http.client import BAD_REQUEST, INTERNAL_SERVER_ERROR
+from pathlib import Path
+
+import pandas as pd
+
+from py_dpm.Exceptions.exceptions import ScriptingError, SemanticError
+from py_dpm.models import OperationVersion, SubCategory, TableVersionCell, ViewReportTypeOperandReferenceInfo
+from py_dpm.db_utils import get_session
+
+
+def format_table_code(code: str):
+    if not code or pd.isna(code):
+        return None
+    code = code.replace(' ', '_')
+    return code
+
+def assing_sign(formula):
+    if "<" in formula:
+        return "negative" # negative
+    elif ">" in formula:
+        return "positive" # positive
+    else:
+        return None
+
+
+def read_external_sign_data(**kwargs):
+    path = Path(__file__).parent / 'utils_report' / 'external_data'
+
+    eba_rules = path / 'EBA_Validation_Rules_2022-12-12.xlsx'
+    sht = '3.2(3.2.1, 3.2.2)'  # wb['v3.1(3.1.1)']  # the last one at 21/06/2022
+
+    eba_df = pd.read_excel(eba_rules, sheet_name=sht)
+    sign_validations = eba_df[eba_df['Type'] == 'Sign']
+    sign_validations['Table Code'] = sign_validations['T1'].map(lambda x: x.replace(' ', '_'))#map(format_table_code)
+    sign_validations["Sign"] = sign_validations["Formula"].map(assing_sign)
+    sign_validations = sign_validations[["Table Code", "Sign", "ID"]]
+    update_sign_db(sign_validations)
+    return sign_validations
+
+
+def update_sign_db(sign_validations):  # TODO: this does not work because DDL restrictions
+    session = get_session()
+    # sign_validations =  read_external_sign_data()
+    neg_tables = sign_validations[sign_validations["Sign"]=='negative']["Table Code"].unique().tolist()
+    pos_tables = sign_validations[sign_validations["Sign"]=='positive']["Table Code"].unique().tolist()
+    all_neg_table_version_ids = TableVersionCell.get_sign_query(session=session, sign=neg_tables)
+    all_pos_table_version_ids = TableVersionCell.get_sign_query(session=session, sign=pos_tables)
+    for table_version in all_neg_table_version_ids:
+        table_version.Sign = "negative"
+        session.commit()
+    for table_version in all_pos_table_version_ids:
+        table_version.Sign = "positive"
+        session.commit()
+
+    # session.commit()
+    session.close()
+
+def _error_to_dict(error):
+    if isinstance(error, SemanticError):
+        status = BAD_REQUEST
+        message_error = error.args[0]
+        error_code = error.args[1]
+    elif isinstance(error, SyntaxError):
+        status = BAD_REQUEST
+        message_error = error.args[0]
+        error_code = "Syntax"
+    elif isinstance(error, ScriptingError):
+        status = BAD_REQUEST
+        message_error = error.args[0]
+        error_code = error.args[1]
+    else:
+        status = INTERNAL_SERVER_ERROR
+        message_error = str(error)
+        now = datetime.now()
+        warnings.warn(f"[{now}] Unhandled exception:\n{error}", RuntimeWarning)
+        error_code = "Other"
+
+    return status, message_error, error_code
+
+
+class ExternalData:
+
+    def __init__(self):
+        self.proposed_rules, self.rejected_rules = self.read_external_data()
+
+    def read_external_data(self, **kwargs):
+        path = Path(__file__).parent / 'utils_report' / 'external_data'
+
+        proposed_rules_path = path / 'out_put_proposed_rules.xlsx'
+        rejected_rules_path = path / 'RejectedRuleProposals.csv'
+        proposed_rules = pd.read_excel(proposed_rules_path)
+        rejected_rules = pd.read_csv(rejected_rules_path, encoding='latin-1')
+        return proposed_rules, rejected_rules
+
+    @staticmethod
+    def get_expression_from_code(code: str):
+        if not code or pd.isna(code):
+            return None
+        sql_session = get_session()
+        expression = OperationVersion.get_operations_from_code(sql_session, code)
+        expression = expression["Expression"].iloc[0]  # TODO check if there is more than one expression take the last one maybe
+        sql_session.close()
+        return expression
+
+    @classmethod
+    def create_report(cls, report_df:pd.DataFrame, info_dict:dict, path:Path, name_report:str):
+        path_info = path / f"{name_report}_info.json"
+        path_validations = path / f"{name_report}_validations.csv"
+        report_df.to_csv(path_validations, index=False)
+        with open(path_info, 'w') as f:
+            f.write(json.dumps(info_dict))
+
+
+class ExternalDataHierarchies(ExternalData):
+
+    def __init__(self):
+        super().__init__()
+        # self.proposed_rules = self.proposed_rules.rename(columns={'HierarchyCode':'Hierarchy Code'})
+        self.proposed_rules = self.proposed_rules.dropna(subset=['Hierarchy Code']).reset_index(drop=True)
+        self.rejected_rules = self.rejected_rules.dropna(subset=['HierarchyCode']).reset_index(drop=True)
+
+    def get_hierarchy_subcategory(self, subcategory_code):
+        proposed_rules, rejected_rules = self.proposed_rules, self.rejected_rules
+
+        proposed_rules = proposed_rules[proposed_rules['Hierarchy Code'] == subcategory_code]
+        rejected_rules = rejected_rules[rejected_rules['HierarchyCode'] == subcategory_code]
+
+        return proposed_rules, rejected_rules
+
+    def compare_all_hierarchies_report(self, validations: list):
+        total_number_of_validations_created = 0
+        total_number_of_validations_proposed = 0
+        total_number_of_validations_rejected = 0
+        total_matches_codes = 0
+        total_duplicated_codes = 0
+        total_codes_dont_match_with_proposed = 0
+        total_missing_codes = 0
+        total_expressions_dont_match_with_proposed = 0
+        sql_session = get_session()
+        subcategory_codes = SubCategory.get_codes(session=sql_session)
+        sql_session.close()
+        subcategory_codes = [subcategory_code[0] for subcategory_code in subcategory_codes]
+        total_info = {}
+        total_merged = pd.DataFrame()
+        for subcategory_code in subcategory_codes:
+            validations_subcategory = [elto for elto in validations if elto['subcategory_code'] == subcategory_code]
+            merged, info = self.compare_hierarchies_report(validations_subcategory, subcategory_code)
+            total_merged = pd.concat([total_merged, merged])
+            for key, value in info.items():
+                total_info[key] = value
+                total_number_of_validations_created += value["number_of_validations_created"]
+                total_number_of_validations_proposed += value["number_of_validations_proposed"]
+                total_number_of_validations_rejected += value["number_of_validations_rejected"]
+                total_matches_codes += len(value["matches_codes(generated_expressions)"])
+                if "errors" in value.keys():
+                    total_duplicated_codes += len(value["errors"]["duplicated_codes"])
+                    total_codes_dont_match_with_proposed += len(value["errors"]["codes_dont_match_with_proposed"])
+                    total_missing_codes += len(value["errors"]["missing_codes"])
+                    total_expressions_dont_match_with_proposed += len(value["errors"]["expressions_dont_match_with_proposed"])
+
+        # aditional actions on total info
+        description = {}
+        description["total_number_of_validations_created"] = total_number_of_validations_created
+        description["total_number_of_validations_proposed"] = total_number_of_validations_proposed
+        description["total_number_of_validations_rejected"] = total_number_of_validations_rejected
+        description["total_matches_codes(generated_expressions)"] = total_matches_codes
+        description["total_duplicated_codes"] = total_duplicated_codes
+        description["total_codes_dont_match_with_proposed"] = total_codes_dont_match_with_proposed
+        description["total_missing_codes"] = total_missing_codes
+        description["total_expressions_dont_match_with_proposed"] = total_expressions_dont_match_with_proposed
+        final_info = {}
+        final_info["resume"] = description
+        final_info["subcategories"] = total_info
+
+        return total_merged, final_info
+
+    def compare_hierarchies_report(self, validations: list, subcategory_code: str):
+        proposed, rejected = self.get_hierarchy_subcategory(subcategory_code=subcategory_code)
+        proposed_specific = pd.DataFrame(columns=['ID', 'Formula', 'ProposedAction', 'Review Action', 'Hierarchy Code']) if proposed.empty else proposed[['ID', 'Formula', 'ProposedAction', 'Review Action', 'Hierarchy Code']]
+        rejected_specific = pd.DataFrame(columns=['ID', 'Formula', 'ProposedAction', 'Review Action', 'Hierarchy Code']) if rejected.empty else rejected[['ID', 'Formula', 'ProposedAction', 'ReviewAction', 'HierarchyCode']].rename(columns={'HierarchyCode':'Hierarchy Code', 'ReviewAction':'Review Action'})
+        info = {}
+        errors = {}
+        errors[subcategory_code] = {}
+        errors[subcategory_code]['duplicated_codes'] = []
+
+        proposed_codes = proposed_specific[~proposed_specific['ID'].isna()]["ID"].unique().tolist()
+        rejected_codes = rejected_specific['ID'].unique().tolist()
+        total_external = pd.concat([proposed_specific, rejected_specific])
+        total_external_codes = total_external['ID'].unique().tolist()
+        # validations_codes
+        validations_codes = []
+
+        [validations_codes.extend(elto['operation_code']) for elto in validations]
+
+        if len(validations_codes) != len(set(validations_codes)):
+            errors[subcategory_code]['duplicated_codes'] = list(set([code for code in validations_codes if validations_codes.count(code) > 1]))
+            validations_codes = list(set(validations_codes))
+
+        errors[subcategory_code]['codes_dont_match_with_proposed'] = [code for code in validations_codes if code not in total_external_codes]  # TODO
+        errors[subcategory_code]['missing_codes'] = [code for code in proposed_codes if code not in validations_codes]
+
+        #create comparative report
+        #first adapt validations
+        validations_to_df = []
+        errors[subcategory_code]['expressions_dont_match_with_proposed'] = []
+
+        for elto in validations:
+            is_duplicated = False
+            if len(elto['operation_code']) == 0:
+                if elto['expression'] not in errors[subcategory_code]['expressions_dont_match_with_proposed']:
+                    errors[subcategory_code]['expressions_dont_match_with_proposed'].append(elto['expression'])
+                validations_to_df.append(
+                    {'code':None, 'expression':elto['expression'], 'status':elto['status'],
+                    'subcategory_code':subcategory_code, 'is_duplicated':is_duplicated
+                    }
+                )
+            if len(set(elto['operation_code'])) > 1:
+                is_duplicated = True
+                errors[subcategory_code]['duplicated_codes'].extend(elto['operation_code'])
+
+            for op_code in elto['operation_code']:
+                validations_to_df.append(
+                    {'code':op_code, 'expression':elto['expression'], 'status':elto['status'],
+                    'subcategory_code':subcategory_code, 'is_duplicated':is_duplicated
+                    }
+                )
+
+        info[subcategory_code] = {'number_of_validations_created':len(validations)}
+        info[subcategory_code]['number_of_validations_proposed'] = len(proposed_specific)
+        info[subcategory_code]['number_of_validations_rejected'] = len(rejected_specific)
+        comparative_report = pd.DataFrame(validations_to_df)
+        if comparative_report.empty:
+            comparative_report = pd.DataFrame(columns=['code', 'expression', 'status', 'subcategory_code', 'is_duplicated'])
+
+        merged = comparative_report.merge(total_external, left_on='code', right_on='ID', how='outer', indicator=True)
+        info[subcategory_code]['matches_codes(generated_expressions)'] = merged[~merged['code'].isna()]["code"].unique().tolist()
+        # external_code, external_expression, proposed action, code, expression, ...
+
+        # if not merged.empty:
+        merged["expression_from_db"] = merged["code"].map(ExternalData.get_expression_from_code)
+        del merged['_merge']
+        errors[subcategory_code]['duplicated_codes'] = list(set(errors[subcategory_code]['duplicated_codes']))
+        info[subcategory_code]["errors"] = errors[subcategory_code]  # TODO
+        proposed_expressions_without_code = proposed_specific[proposed_specific['ID'].isna()]["Formula"].unique().tolist()
+        if proposed_expressions_without_code:
+            info[subcategory_code]['proposed_expressions_without_code'] = proposed_expressions_without_code
+        info[subcategory_code]['aditional info']=[{elto["expression"]:elto["aproximated_operations"]} for elto in validations if not elto["operation_code"]]
+        return merged, info
+
+    # @staticmethod
+    # def create_report(report_df:pd.DataFrame, info_dict:dict, subcategory_code:str=None):
+    #     name_report = subcategory_code if subcategory_code else 'all_subcategories'
+    #     path = Path(__file__).parent / 'utils_report' / 'hierarchies_report'
+    #     path_errors = path / f"{name_report}_info.json"
+    #     path_validations = path / f"{name_report}_validations.csv"
+    #     report_df.to_csv(path_validations, index=False)
+    #     with open(path_errors, 'w') as f:
+    #         f.write(json.dumps(info_dict))
+
+    @classmethod
+    def create_report(cls, report_df:pd.DataFrame, info_dict:dict, subcategory_code:str=None):
+        name_report = subcategory_code if subcategory_code else 'all_subcategories'
+        path = Path(__file__).parent / 'utils_report' / 'hierarchies_report'
+        super().create_report(report_df, info_dict, path, name_report)
+
+
+class ExternalDataSign(ExternalData):
+
+    def __init__(self):
+        super().__init__()
+        self.proposed_rules = self.proposed_rules[self.proposed_rules['Type']=="Sign"]
+        self.rejected_rules = self.rejected_rules[self.rejected_rules['Type']=="Sign"]
+
+
+    def match_code(self, validations: list, report_type: str):
+        """
+        """
+        for elto in validations:
+            total_cells_id = elto["cell_data"]
+            sql_session = get_session()
+            matched_codes = ViewReportTypeOperandReferenceInfo.get_cell_report_operations(sql_session, total_cells_id, report_type)
+            elto["operation_code"] = matched_codes
+            sql_session.close()
+            # new version end
+
+        return validations
+
+    def compare_all_sign_report(self, validations: list):
+        proposed, rejected = self.proposed_rules, self.rejected_rules
+        proposed_specific = pd.DataFrame(columns=['ID', 'Formula', 'ProposedAction', 'Review Action', 'Framework']) if proposed.empty else proposed[['ID', 'Formula', 'ProposedAction', 'Review Action', 'Framework']]
+        rejected_specific = pd.DataFrame(columns=['ID', 'Formula', 'ProposedAction', 'Review Action']) if rejected.empty else rejected[['ID', 'Formula', 'ProposedAction', 'ReviewAction']].rename(columns={'ReviewAction':'Review Action'})
+        info = {}
+        errors = {}
+
+        proposed_codes = proposed_specific[~proposed_specific['ID'].isna()]["ID"].unique().tolist()
+        rejected_codes = rejected_specific['ID'].unique().tolist()
+        total_external = pd.concat([proposed_specific, rejected_specific])
+        total_external_codes = total_external['ID'].unique().tolist()
+        # validations_codes
+        validations_codes = []
+
+        [validations_codes.extend(elto['operation_code']) for elto in validations]
+
+        if len(validations_codes) != len(set(validations_codes)):
+            errors['duplicated_codes'] = list(set([code for code in validations_codes if validations_codes.count(code) > 1]))
+            validations_codes = list(set(validations_codes))
+
+        # no match expressions
+        errors['expressions_dont_match_with_proposed'] = [elto['expression'] for elto in validations if len(elto['operation_code']) == 0]
+        # missing codes
+        errors['missing_codes'] = [code for code in total_external_codes if code not in validations_codes]
+        # exceded codes
+        errors['exceded_codes'] = [code for code in validations_codes if code not in total_external_codes]
+
+        validations_to_df = []
+        for elto in validations:
+            if len(elto['operation_code']) == 0:
+                validations_to_df.append(
+                    {'code':None, 'expression':elto['expression'], 'status':elto['status'],
+                    'table':None
+                    }
+                )
+            elif len(elto['operation_code']) == 1:
+                validations_to_df.append(
+                    {'code':elto['operation_code'][0], 'expression':elto['expression'], 'status':elto['status'],
+                    'table':None
+                    }
+                    )
+            else:
+                for code in elto['operation_code']:
+                    validations_to_df.append(
+                        {'code':code, 'expression':elto['expression'], 'status':elto['status'],
+                        'table':None
+                        }
+                        )
+        comparative_report = pd.DataFrame(validations_to_df)
+        if comparative_report.empty:
+            comparative_report = pd.DataFrame(columns=['code', 'expression', 'status', 'subcategory_code', 'is_duplicated'])
+
+        merged = comparative_report.merge(total_external, left_on='code', right_on='ID', how='outer', indicator=True)
+        merged["expression_from_db"] = merged["code"].map(ExternalData.get_expression_from_code)
+        del merged['_merge']
+        # total_merged, total_info = proposed_specific, rejected_specific
+
+        return merged, errors
+
+    @classmethod
+    def create_report(cls, report_df:pd.DataFrame, info_dict:dict):
+        name_report = 'signes'
+        path = Path(__file__).parent / 'utils_report' / 'signes_report'
+        super().create_report(report_df, info_dict, path, name_report)
+
+class ExternalDataExistence(ExternalData):
+
+    def __init__(self):
+        super().__init__()
+        self.proposed_rules = self.proposed_rules[self.proposed_rules['Type']=="Existence"]
+        self.rejected_rules = self.rejected_rules[self.rejected_rules['Type']=="Existence"]
+
+    def compare_all_existence_report(self, validations: list):
+        pass

py_dpm/ValidationsGeneration/VariantsProcessor.py

@@ -0,0 +1,265 @@
+import copy
+import re
+
+import pandas as pd
+
+from py_dpm.AST.ASTObjects import VarID, WithExpression
+from py_dpm.AST.ASTTemplate import ASTTemplate
+from py_dpm.Exceptions import exceptions
+from py_dpm.models import ModuleVersionComposition, TableGroup, TableGroupComposition, TableVersion, ViewDatapoints
+from py_dpm.Utils.tokens import EXPRESSION, STATUS, STATUS_CORRECT, STATUS_INCOMPLETE, STATUS_INCORRECT, VALIDATION_CODE
+from py_dpm.data_handlers import filter_all_data
+
+cell_components = ['table', 'rows', 'cols', 'sheets']
+TABLE_ID = 'TableID'
+MODULE_VID = 'ModuleVID'
+TABLE_CODE = 'Code'
+GROUP = 'group'
+MODULE_VERSION_ID = 'module_version_id'
+MODULE_CODE = 'module_code'
+
+
+class VariantsProcessorChecker(ASTTemplate):
+    def __init__(self, ast):
+        super().__init__()
+        self._is_variant = False
+        self.visit(ast)
+
+    @property
+    def is_variant(self):
+        return self._is_variant
+
+    def visit_VarID(self, node: VarID):
+        if node.is_table_group:
+            self._is_variant = True
+
+
+class VariantsProcessor(ASTTemplate):
+    """
+    Class to generate individual validations from a validation defined on groups.
+
+    :parameter expression: DPM-XL expression.
+    :parameter ast: Abstract Syntax Tree of expression.
+    :parameter session: SQLAlchemy Session to be used to connect to the DB.
+    :parameter validation_code: Code of parent validation.
+    """
+
+    def __init__(self, expression, ast, session, validation_code, release_id):
+        super().__init__()
+        self.expression = expression
+        self.AST = ast
+        self.session = session
+        self.validation_code = validation_code
+        self.current_validation = 1
+        self.partial_selection = None
+        self.release_id = release_id
+
+        self.table_data = {}
+        self.group_tables = []
+        self.table_groups_compositions = {}
+        self.tables_without_cells = []
+        self.children_suffix = None
+        self.equal_children_suffix = True
+
+        self.visit(self.AST)
+
+    def check_if_table_has_cells(self, table_code, rows, cols, sheets):
+        """
+        Method to check if an individual table has cells for rows, columns and sheets of validation.
+        :param table_code: code of table.
+        :param rows: rows.
+        :param cols: columns.
+        :param sheets: sheets.
+        :return: True if table has cells, False otherwise.
+        """
+        if table_code not in self.table_data:
+            datapoints = ViewDatapoints.get_table_data(session=self.session, table=table_code, release_id=self.release_id)
+            self.table_data[table_code] = datapoints
+        else:
+            datapoints = self.table_data[table_code]
+        data = filter_all_data(data=datapoints, table_code=table_code, rows=rows, cols=cols, sheets=sheets)
+        return not data.empty
+
+    def check_table_from_module(self, node, table_module_id, is_abstract):
+        """
+        Method to get table_versions of a table group when there are cells for those tables.
+        :param node: var_id node
+        :param table_module_id: module id
+        :param is_abstract: flag that represents if table is abstract
+        :return: table version if table version has cells, None otherwise.
+        """
+
+        table_versions = TableVersion.get_tables_versions_of_table_group_compositions(session=self.session,
+                                                                                      table_id=table_module_id,
+                                                                                      is_abstract=is_abstract,
+                                                                                      release_id=self.release_id)
+
+        for table_version in table_versions:
+            table_with_cells = self.check_if_table_has_cells(table_version.Code, node.rows, node.cols, node.sheets)
+            if table_with_cells:
+                return table_version
+            else:
+                if table_version.TableID not in self.tables_without_cells:
+                    self.tables_without_cells.append(table_version.TableID)
+        return None
+
+    def generate_child_expression(self, group_code, table_code):
+        """
+        Method to replace group_code by table_code in order to generate new validations.
+        :param group_code: Group code.
+        :param table_code: Table code.
+        """
+        groups = re.search("(" + re.escape(group_code) + '[^.$]' + ")", self.expression)
+        if f"g{group_code}" in self.expression and groups:
+            group = groups.group(0)
+            suffix = table_code[-(len(table_code)-len(group_code)):]
+            if group:
+                if not self.children_suffix:
+                    self.children_suffix = suffix
+                else:
+                    if self.children_suffix != suffix:
+                        self.equal_children_suffix = False
+                        return
+                self.expression = re.sub(re.escape(f"g{group_code}") + '[^.$]', f"t{table_code}" + group[-1], self.expression)
+
+    def generate_child_expressions(self):
+        """
+        Method to generate all the individual validations.
+        :return: Expressions and expressions with errors.
+        """
+        if self.group_tables:
+            final_expressions = []
+            final_expressions_with_errors = []
+            df_tables = pd.DataFrame.from_records(self.group_tables)
+            df_tables = df_tables[~df_tables[TABLE_ID].isin(self.tables_without_cells)]
+            df_tables.drop_duplicates(inplace=True)
+
+            table_ids = df_tables[TABLE_ID].tolist()
+            modules_df = ModuleVersionComposition.get_modules_from_table_ids(session=self.session, table_ids=table_ids,
+                                                                             release_id=self.release_id)
+
+            data = pd.merge(df_tables, modules_df, on=[TABLE_ID])
+
+            expression = copy.deepcopy(self.expression)
+            for module, modules_tables in data.groupby(MODULE_VID):
+                modules_tables.apply(lambda x: self.generate_child_expression(x[GROUP], x[TABLE_CODE]), axis=1)
+                module_code = modules_tables[MODULE_CODE].unique().tolist()[0]
+                if len(modules_tables) < len(self.table_groups_compositions):
+                    final_expressions_with_errors.append(
+                        {EXPRESSION: self.expression, MODULE_VERSION_ID: module, MODULE_CODE: module_code})
+                else:
+                    if self.equal_children_suffix:
+                        final_expressions.append(
+                            {EXPRESSION: self.expression, MODULE_VERSION_ID: module, MODULE_CODE: module_code})
+                    else:
+                        final_expressions_with_errors.append(
+                            {EXPRESSION: self.expression, MODULE_VERSION_ID: module, MODULE_CODE: module_code})
+                        self.equal_children_suffix = True
+                self.expression = copy.deepcopy(expression)
+                self.children_suffix = None
+
+            return final_expressions, final_expressions_with_errors
+        else:
+            raise exceptions.SemanticError("5-2-1")
+
+    def create_validation(self, expression_info, status):
+        """
+        Method to centralize creation of validations given expression and status of new validation
+        :param expression_info: Dictionary with information about expression, module_vid and module_code of validation
+        :param status: Status of validation
+        :return: Validation with code, expression and status
+        """
+
+        validation_code = self.generate_child_validation_code() if status == STATUS_CORRECT else None
+
+        validation = {VALIDATION_CODE: validation_code,
+                      EXPRESSION: expression_info[EXPRESSION],
+                      STATUS: status,
+                      MODULE_VERSION_ID: expression_info[MODULE_VERSION_ID],
+                      MODULE_CODE: expression_info[MODULE_CODE]}
+        return validation
+
+    def create_validation_new_format(self, expressions_dict):
+        """
+        Method to centralize creation of validations given expression and status of new validation
+        :param expression_info: Dictionary with information about expression, module_vid and module_code of validation
+        :param status: Status of validation
+        :return: Validation with code, expression and status
+        """
+
+        correct_expressions = expressions_dict[STATUS_CORRECT]
+        incomplete_expressions = expressions_dict[STATUS_INCOMPLETE]
+        incorrect_expressions = expressions_dict[STATUS_INCORRECT]
+        unique_correct_expressions = pd.DataFrame.from_records(correct_expressions)['expression'].unique().tolist() if correct_expressions else []
+        unique_incomplete_expressions = pd.DataFrame.from_records(incomplete_expressions)['expression'].unique().tolist() if incomplete_expressions else []
+        unique_incorrect_expressions = pd.DataFrame.from_records(incorrect_expressions)['expression'].unique().tolist() if incorrect_expressions else []
+        validations = []
+        if correct_expressions:
+            v = self._aux_create_validation_new_format(correct_expressions, unique_correct_expressions, STATUS_CORRECT)
+            validations.extend(v)
+        if incomplete_expressions:
+            v = self._aux_create_validation_new_format(incomplete_expressions, unique_incomplete_expressions, STATUS_INCOMPLETE)
+            validations.extend(v)
+        if incorrect_expressions:
+            v = self._aux_create_validation_new_format(incorrect_expressions, unique_incorrect_expressions, STATUS_INCORRECT)
+            validations.extend(v)
+
+        return validations
+
+    def _aux_create_validation_new_format(self, expressions, unique_expressions, status):
+        validations = []
+        for expr in unique_expressions:
+            aux = {}
+            aux[EXPRESSION] = expr
+            aux[STATUS] = status
+            if status == STATUS_CORRECT:
+                aux[VALIDATION_CODE] = self.generate_child_validation_code()
+            else:
+                aux[VALIDATION_CODE] = None
+            aux['scopes'] = []
+            for elto in expressions:
+                if elto[EXPRESSION] == aux[EXPRESSION]:
+                    aux['scopes'].append(
+                        {"module_versions_ids": [elto[MODULE_VERSION_ID]],
+                        "module_code": elto[MODULE_CODE]
+                        }
+                    )
+            validations.append(aux)
+        return validations
+
+    def generate_child_validation_code(self):
+        """
+        Method to calculate validation codes for new validations.
+        :return: New code for the individual validation.
+        """
+        child_code = f"{self.validation_code}-{self.current_validation}"
+        self.current_validation += 1
+        return child_code
+
+    def visit_WithExpression(self, node: WithExpression):
+        self.partial_selection = node.partial_selection
+        self.visit(node.partial_selection)
+        self.visit(node.expression)
+
+    def visit_VarID(self, node: VarID):
+        if self.partial_selection:
+            for attribute in cell_components:
+                if not getattr(node, attribute, False) and hasattr(self.partial_selection, attribute):
+                    setattr(node, attribute, getattr(self.partial_selection, attribute))
+
+        if node.is_table_group or self.partial_selection.is_table_group:
+            if node.table:
+                if node.table not in self.table_groups_compositions:
+                    group: TableGroup = TableGroup.get_group_from_code(session=self.session, group_code=node.table)
+                    if not group:
+                        raise exceptions.SemanticError("1-6", table_group=node.table)
+                    table_groups_compositions = TableGroupComposition.get_from_parent_table_code(code=node.table, session=self.session)
+                    self.table_groups_compositions[node.table] = table_groups_compositions
+                else:
+                    table_groups_compositions = self.table_groups_compositions[node.table]
+
+                for table_id, is_abstract in table_groups_compositions:
+                    table_version = self.check_table_from_module(node, table_id, is_abstract)
+                    if table_version:
+                        self.group_tables.append(
+                            {GROUP: node.table, TABLE_CODE: table_version.Code, TABLE_ID: table_version.TableID})