pydpm_xl 0.1.10__py3-none-any.whl

py_dpm/Utils/tokens.py

@@ -0,0 +1,172 @@
+# Comparison operators.
+EQ = '='
+NEQ = '!='
+GT = '>'
+GTE = '>='
+LT = '<'
+LTE = '<='
+IN = 'in'
+ISNULL = 'isnull'
+MATCH = 'match'
+
+# Numeric operators
+PLUS = '+'
+MINUS = '-'
+MULT = '*'
+DIV = '/'
+ABS = 'abs'
+EXP = 'exp'
+LN = 'ln'
+SQRT = 'sqrt'
+POW = 'power'
+LOG = 'log'
+MAX = 'max'
+MIN = 'min'
+
+# Boolean operators.
+AND = 'and'
+OR = 'or'
+XOR = 'xor'
+NOT = 'not'
+
+# Conditional operators.
+IF = "if"
+NVL= "nvl"
+
+# Clause operators
+WHERE = 'where'
+RENAME = 'rename'
+GET = 'get'
+
+# Aggregation operators
+MAX_AGGR = 'max_aggr'
+MIN_AGGR = 'min_aggr'
+SUM = 'sum'
+COUNT = 'count'
+AVG = 'avg'
+MEDIAN = 'median'
+
+# String operators
+LENGTH = "len"
+CONCATENATE = "&"
+
+# Time operators
+TIME_SHIFT = 'time_shift'
+
+# Conditional operators
+FILTER = 'filter'
+
+# key Components types
+DPM = "DPM"
+STANDARD = "Standard"
+
+# Standard key names
+ROW = 'r'
+COLUMN = 'c'
+SHEET = 's'
+FACT = 'f'
+
+# Indexes
+INDEX_X = 'x'
+INDEX_Y = 'y'
+INDEX_Z = 'z'
+
+# Cell components
+ROW_CODE = 'row_code'
+COLUMN_CODE = 'column_code'
+SHEET_CODE = 'sheet_code'
+TABLE_CODE = 'table_code'
+CELL_COMPONENTS = [ROW_CODE, COLUMN_CODE, SHEET_CODE]
+
+# Generated validations status
+STATUS = 'status'
+STATUS_CORRECT = 'Correct'
+STATUS_INCORRECT = 'Incorrect'
+STATUS_INCOMPLETE = 'Incomplete'
+STATUS_UNKNOWN = 'Unknown'
+TABLE_VERSION_ID = 'table_version_id'
+
+# Generated validations constants
+ITEM_ID = 'ItemID'
+PARENT_ITEM_ID = 'ParentItemID'
+VARIABLE_VID = 'variable_vid'
+ARITHMETIC_OPERATOR_ID = 'ArithmeticOperatorID'
+ORDER = 'Order'
+
+TABLE_CODE_LEFT = TABLE_CODE + '_left'
+ROW_CODE_LEFT = ROW_CODE + '_left'
+COLUMN_CODE_LEFT = COLUMN_CODE + '_left'
+SHEET_CODE_LEFT = SHEET_CODE + '_left'
+
+TABLE_CODE_RIGHT = TABLE_CODE + '_right'
+ROW_CODE_RIGHT = ROW_CODE + '_right'
+COLUMN_CODE_RIGHT = COLUMN_CODE + '_right'
+SHEET_CODE_RIGHT = SHEET_CODE + '_right'
+VARIABLE_PROPERTY_ID = 'variable_property_id'
+
+CONTEXT_PROPERTY = 'context_property'
+OPERATOR_ID = 'OperatorID'
+SYMBOL = 'Symbol'
+ARITHMETIC_OPERATOR_SYMBOL = 'arithmetic_operator_symbol'
+COMPARISON_OPERATOR_ID = 'ComparisonOperatorID'
+COMPARISON_OPERATOR_SYMBOL = 'comparison_operator_symbol'
+
+OPERATOR = 'operator'
+IS_DEFAULT_ITEM = 'IsDefaultItem'
+PROPERTY_ID = 'property_id'
+CONTEXT_ITEM = 'context_item'
+CONTEXT_ITEM_ID = 'context_item_id'
+METRIC_PROPERTY_ID = 'metric_property_id'
+CONTEXT_PROPERTY_ID = 'context_property_id'
+SUBCATEGORY_PROPERTY = 'subcategory_property'
+
+VALIDATION_CODE = 'validation_code'
+TABLE_GROUP_CHILD_TYPE = 'tableGroup_child'
+EXPRESSION = 'expression'
+DUPLICATE_VARIABLES = 'duplicate_variables'
+CELL_ID = 'cell_id'
+SUBCATEGORY_ID = 'subcategory_id'
+SUBCATEGORY_CODE = 'subcategory_code'
+PARENT_ID = 'parent_id'
+LEFT_CELL_IDS = 'left_cell_ids'
+KEY_ID = 'KeyID'
+
+# Data types
+PER = "p"
+
+INPUTS = 'inputs'
+OUTPUTS = 'outputs'
+
+# Prefix
+TABLE_GROUP_PREFIX = 'g'
+
+# Modules
+INTRA_MODULE = 'intra-module'
+CROSS_MODULE = 'cross-module'
+REPEATED_INTRA_MODULE = 'repeated-intra-module'
+REPEATED_CROSS_MODULE = 'repeated-cross-module'
+
+# Operation scope constants
+WARNING_SEVERITY = 'warning'
+
+FILING_INDICATOR = 'filingIndicator'
+OP_VERSION_ID = 'op_version_id'
+
+# Report types
+HIERARCHY_REPORT = 'hierarchy_report'
+SIGN_REPORT = 'sign_report'
+EXISTENCE_REPORT = 'existence_report'
+HIERARCHY = 'hierarchy'
+SIGN = 'sign'
+EXISTENCE = 'existence'
+#sign
+POSITIVE = 'positive'
+NEGATIVE = 'negative'
+
+# Web service constants
+CODE = 'code'
+ERROR = 'error'
+ERROR_CODE = 'error_code'
+VALIDATIONS = 'validations'
+VARIABLES = 'variables'
+VALIDATION_TYPE = 'validation_type'

py_dpm/Utils/utils.py

@@ -0,0 +1,2 @@
+START_RELEASE_ID_EBA = None
+START_RELEASE_ID_EIOPA = 1020000001

py_dpm/ValidationsGeneration/PropertiesConstraintsProcessor.py

@@ -0,0 +1,190 @@
+import pandas as pd
+
+from py_dpm.AST.ASTObjects import AggregationOp, BinOp, ComplexNumericOp, CondExpr, FilterOp, GetOp, PropertyReference, RenameOp, Scalar, \
+    TimeShiftOp, UnaryOp, VarID, WhereClauseOp
+from py_dpm.AST.ASTTemplate import ASTTemplate
+from py_dpm.Exceptions import exceptions
+from py_dpm.models import ItemCategory, ViewDatapoints
+from py_dpm.Utils.ValidationsGenerationUtils import ValidationsGenerationUtils
+from py_dpm.Utils.tokens import *
+
+ALLOWED_OPERATORS = [MATCH, IN, EQ, NEQ, GT, GTE, LT, LTE, LENGTH, CONCATENATE]
+
+
+def _check_property_constraint_exists(signature: str, session):
+    if ":" in signature:
+        property_query = ItemCategory.get_property_from_signature(signature, session)
+    else:
+        property_query = ItemCategory.get_property_from_code(signature, session)
+    if property_query is None:
+        return False
+    return True
+
+
+class PropertiesConstraintsChecker(ASTTemplate):
+    def __init__(self, ast, session):
+        super().__init__()
+        self.has_property = False
+        self.has_table = False
+        self.session = session
+        self.visit(ast)
+
+    @property
+    def is_property_constraint(self):
+        if self.has_table:
+            return False
+        return self.has_property
+
+    def visit_PropertyReference(self, node: PropertyReference):
+        # Optional
+        if not ":" in node.code:
+            pass  # signature should have : to be a property constraint
+        signature = node.code
+        # look for property in models
+        property_query = ItemCategory.get_property_from_signature(signature, self.session)
+        if property_query is None:
+            raise exceptions.SemanticError("5-1-4", ref=signature)
+        self.has_property = True
+
+    def visit_VarID(self, node: VarID):
+        if node.table:
+            self.has_table = True
+
+    def visit_Scalar(self, node: Scalar):
+        signature = node.item
+        if not self.has_property:
+            if getattr(node, "scalar_type", None) == "Item":
+                # go to models and check if item exists and is a property
+                property_query = ItemCategory.get_property_from_signature(signature, self.session)
+                if property_query:
+                    self.has_property = True
+                # other assumption could be always first scalar is a property but this is not true
+                # self.has_property = True
+        else:
+            other_property_query = ItemCategory.get_property_from_signature(signature, self.session)
+            if other_property_query:
+                raise exceptions.SemanticError("5-1-2")
+
+
+class PropertiesConstraintsProcessor(ASTTemplate):
+    """
+    Class to generate individual validations from properties constraints
+
+    :parameter expression: DPM-XL expression.
+    :parameter ast: Abstract Syntax Tree of expression.
+    :parameter validation_code: Code of parent validation.
+    :parameter session: SQLAlchemy Session to be used to connect to the DB.
+    """
+
+    def __init__(self, expression, ast, validation_code, session, release_id):
+        super().__init__()
+        self.expression = expression
+        self.AST = ast
+        self.validation_code = validation_code
+        self.session = session
+        self.current_validation = 1
+        self.property_constraint = None
+        self.release_id = release_id
+        self.new_expressions = []
+        self.visit(self.AST)
+
+    def generate_validations(self):
+        """
+        Generates individual validations using the extracted property constraint in the Abstract Syntax Tree
+        """
+        if not self.property_constraint:
+            raise exceptions.SemanticError("5-1-1")
+
+        item_category = ItemCategory.get_property_from_signature(
+                signature=self.property_constraint, session=self.session, release_id=self.release_id)
+        if item_category is None:
+            raise exceptions.SemanticError("1-7", property_code=self.property_constraint)
+        variables: pd.DataFrame = ViewDatapoints.get_from_property(self.session, item_category.ItemID, self.release_id)
+        for table_code, group_df in variables.groupby(['table_code']):
+            datapoints = ViewDatapoints.get_table_data(session=self.session, table=str(table_code))
+            self.generate_expressions(table_code, group_df, datapoints)
+
+    def generate_expressions(self, table_code, data, datapoints_table):
+        """
+        Generates new expressions getting their operands by grouping the cells
+        :param table_code: code of the operand table
+        :param data: dataframe with operand datapoints
+        :param datapoints_table: table datapoints
+        """
+        groups = ValidationsGenerationUtils.group_cells(datapoints_variable=data,
+                                                        datapoints_table=datapoints_table)
+        for rows, cols, sheets in groups:
+            operand = ValidationsGenerationUtils.write_cell(table_code, rows, cols, sheets)
+            new_expression = self.expression
+            new_expression = new_expression.replace(f"[{self.property_constraint}]", operand)
+            self.new_expressions.append(new_expression)
+
+    def create_validation(self, expression, status):
+        """
+        Creates a dictionary to represent a validation from expression and status information
+        :param expression: Expression of validation
+        :param status: Status of validation
+        :return a dictionary with validation_code, expression and status
+        """
+        validation_code = None
+        if status == STATUS_CORRECT:
+            validation_code = f"{self.validation_code}-{self.current_validation}"
+            self.current_validation += 1
+        return {
+            VALIDATION_CODE: validation_code,
+            EXPRESSION: expression,
+            STATUS: status
+        }
+
+    def visit_PropertyReference(self, node: PropertyReference):
+        if not self.property_constraint:
+            self.property_constraint = node.code
+            signature = node.code
+            if not _check_property_constraint_exists(signature, self.session):
+                raise exceptions.SemanticError("5-1-4", ref=signature)
+        else:
+            raise exceptions.SemanticError("5-1-2")
+
+    def visit_Scalar(self, node: Scalar):
+        if getattr(node, "scalar_type", None) == "Item":
+            signature = node.item
+            property_query = ItemCategory.get_property_from_signature(signature, self.session)
+            if property_query:
+                if not self.property_constraint:
+                    self.property_constraint = signature
+
+    def visit_BinOp(self, node: BinOp):
+        if node.op not in ALLOWED_OPERATORS:
+            raise exceptions.SemanticError("5-1-3", operator=node.op)
+
+        self.visit(node.left)
+        self.visit(node.right)
+
+    def visit_UnaryOp(self, node: UnaryOp):
+        if node.op not in ALLOWED_OPERATORS:
+            raise exceptions.SemanticError("5-1-3", operator=node.op)
+        self.visit(node.operand)
+
+    def visit_CondExpr(self, node: CondExpr):
+        raise exceptions.SemanticError("5-1-3", operator=IF)
+
+    def visit_AggregationOp(self, node: AggregationOp):
+        raise exceptions.SemanticError("5-1-3", operator=node.op)
+
+    def visit_RenameOp(self, node: RenameOp):
+        raise exceptions.SemanticError("5-1-3", operator=RENAME)
+
+    def visit_TimeShiftOp(self, node: TimeShiftOp):
+        raise exceptions.SemanticError("5-1-3", operator=TIME_SHIFT)
+
+    def visit_FilterOp(self, node: FilterOp):
+        raise exceptions.SemanticError("5-1-3", operator=FILTER)
+
+    def visit_WhereClauseOp(self, node: WhereClauseOp):
+        raise exceptions.SemanticError("5-1-3", operator=WHERE)
+
+    def visit_GetOp(self, node: GetOp):
+        raise exceptions.SemanticError("5-1-3", operator=GET)
+
+    def visit_ComplexNumericOp(self, node: ComplexNumericOp):
+        raise exceptions.SemanticError("5-1-3", operator=node.op)