pydpm_xl 0.1.10__py3-none-any.whl → 0.1.39rc23__py3-none-any.whl

py_dpm/AST/check_operands.py

@@ -1,20 +1,46 @@
 from abc import ABC
 
 import pandas as pd
-
-from py_dpm.AST.ASTObjects import Dimension, OperationRef, PersistentAssignment, PreconditionItem, \
-    Scalar, TemporaryAssignment, VarID, VarRef, WhereClauseOp, WithExpression
+import warnings
+
+# Suppress pandas UserWarning about SQLAlchemy connection types
+warnings.filterwarnings("ignore", message=".*pandas only supports SQLAlchemy.*")
+
+from py_dpm.AST.ASTObjects import (
+    Dimension,
+    GetOp,
+    OperationRef,
+    PersistentAssignment,
+    PreconditionItem,
+    Scalar,
+    TemporaryAssignment,
+    VarID,
+    VarRef,
+    WhereClauseOp,
+    WithExpression,
+)
 from py_dpm.AST.ASTTemplate import ASTTemplate
 from py_dpm.AST.WhereClauseChecker import WhereClauseChecker
 from py_dpm.DataTypes.ScalarTypes import Integer, Mixed, Number, ScalarFactory
 from py_dpm.Exceptions import exceptions
 from py_dpm.Exceptions.exceptions import SemanticError
-from py_dpm.models import ItemCategory, Operation, VariableVersion, ViewDatapoints, \
-    ViewKeyComponents, ViewOpenKeys
+from py_dpm.models import (
+    ItemCategory,
+    Operation,
+    VariableVersion,
+    ViewDatapoints,
+    ViewKeyComponents,
+    ViewOpenKeys,
+    Table,
+    TableVersion,
+    TableVersionHeader,
+    Header,
+    filter_by_release,
+)
 from py_dpm.Utils.operands_mapping import generate_new_label, set_operand_label
 from py_dpm.data_handlers import filter_all_data
 
-operand_elements = ['table', 'rows', 'cols', 'sheets', 'default', 'interval']
+operand_elements = ["table", "rows", "cols", "sheets", "default", "interval"]
 
 
 def _create_operand_label(node):
@@ -25,12 +51,12 @@ def _create_operand_label(node):
 def _modify_element_info(new_data, element, table_info):
     if new_data is None and table_info[element] is None:
         pass
-    elif table_info[element] == ['*']:
+    elif table_info[element] == ["*"]:
         pass
     elif new_data is not None and table_info[element] is None:
         table_info[element] = new_data
 
-    elif new_data == ['*']:
+    elif new_data == ["*"]:
         # We have already all data available
         table_info[element] = new_data
 
@@ -48,14 +74,61 @@ def _modify_table(node, table_info):
 
 
 def format_missing_data(node):
-    rows = ', '.join([f"r{x}" for x in node.rows]) if node.rows else None
-    cols = ', '.join([f"c{x}" for x in node.cols]) if node.cols else None
-    sheets = ', '.join([f"s{x}" for x in node.sheets]) if node.sheets else None
+    rows = ", ".join([f"r{x}" for x in node.rows]) if node.rows else None
+    cols = ", ".join([f"c{x}" for x in node.cols]) if node.cols else None
+    sheets = ", ".join([f"s{x}" for x in node.sheets]) if node.sheets else None
     op_pos = [node.table, rows, cols, sheets]
     cell_exp = ", ".join(x for x in op_pos if x is not None)
     raise exceptions.SemanticError("1-2", cell_expression=cell_exp)
 
 
+def _has_range_syntax(values):
+    """Check if a list of values contains range syntax (e.g., '0010-0080')."""
+    if not values or not isinstance(values, list):
+        return False
+    return any("-" in str(v) for v in values if v and v != "*")
+
+
+def _expand_ranges_from_data(node, node_data):
+    """
+    Expand range-type values in VarID node's rows/cols/sheets to actual codes from the database.
+
+    When a VarID has range syntax like ['0010-0080'], this function replaces it with
+    the actual codes found in node_data (e.g., ['0010', '0020', '0030', ...]).
+
+    This ensures adam-engine receives scalar values in the JSON output instead of
+    list-type ranges which it cannot parse.
+
+    Args:
+        node: VarID AST node with rows, cols, sheets attributes
+        node_data: DataFrame containing the actual cell data with row_code, column_code, sheet_code
+    """
+    if node_data is None or node_data.empty:
+        return
+
+    # Expand rows if they contain range syntax
+    if _has_range_syntax(node.rows):
+        actual_rows = node_data["row_code"].dropna().unique().tolist()
+        if actual_rows:
+            # Sort to maintain consistent ordering
+            actual_rows = sorted(actual_rows)
+            node.rows = actual_rows
+
+    # Expand cols if they contain range syntax
+    if _has_range_syntax(node.cols):
+        actual_cols = node_data["column_code"].dropna().unique().tolist()
+        if actual_cols:
+            actual_cols = sorted(actual_cols)
+            node.cols = actual_cols
+
+    # Expand sheets if they contain range syntax
+    if _has_range_syntax(node.sheets):
+        actual_sheets = node_data["sheet_code"].dropna().unique().tolist()
+        if actual_sheets:
+            actual_sheets = sorted(actual_sheets)
+            node.sheets = actual_sheets
+
+
 class OperandsChecking(ASTTemplate, ABC):
     def __init__(self, session, expression, ast, release_id, is_scripting=False):
         self.expression = expression
@@ -69,7 +142,14 @@ class OperandsChecking(ASTTemplate, ABC):
         self.items = []
         self.preconditions = False
         self.dimension_codes = []
+        self.dimension_nodes = (
+            []
+        )  # Store references to Dimension nodes for property_id enrichment
         self.open_keys = None
+        self.getop_components = []  # Store GetOp component codes for property_id lookup
+        self.getop_nodes = (
+            []
+        )  # Store references to GetOp nodes for property_id enrichment
 
         self.operations = []
         self.operations_data = None
@@ -83,107 +163,228 @@ class OperandsChecking(ASTTemplate, ABC):
         self.check_items()
         self.check_tables()
         self.check_dimensions()
+        self.check_getop_components()
 
         self.check_operations()
 
     def _check_header_present(self, table, header):
-        if (self.partial_selection is not None and self.partial_selection.table == table and
-                getattr(self.partial_selection, header) is not None):
+        if (
+            self.partial_selection is not None
+            and self.partial_selection.table == table
+            and getattr(self.partial_selection, header) is not None
+        ):
             return
         for node in self.operands[table]:
             if getattr(node, header) is None:
-                if header == 'cols':
-                    header = 'columns'
+                if header == "cols":
+                    header = "columns"
                 raise exceptions.SemanticError("1-20", header=header, table=table)
 
     def check_headers(self):
         table_codes = list(self.tables.keys())
         if len(table_codes) == 0:
             return
-        query = """
-        SELECT DISTINCT tv.Code, tv.StartReleaseID, tv.EndReleaseID, h.Direction, t.HasOpenRows, t.HasOpenColumns, t.HasOpenSheets
-        FROM [Table] AS t
-        INNER JOIN TableVersion tv ON t.TableID = tv.TableID
-        INNER JOIN TableVersionHeader tvh ON tv.TableVID = tvh.TableVID
-        INNER JOIN Header h ON h.HeaderID = tvh.HeaderID
-        """
-        codes = [f"{code!r}" for code in table_codes]
-        query += f"WHERE tv.Code IN ({', '.join(codes)})"
-        query += "AND tv.EndReleaseID is null"
-        df_headers = pd.read_sql(query, self.session.connection())
+
+        # Build ORM query
+        query = (
+            self.session.query(
+                TableVersion.code.label("Code"),
+                TableVersion.startreleaseid.label("StartReleaseID"),
+                TableVersion.endreleaseid.label("EndReleaseID"),
+                Header.direction.label("Direction"),
+                Table.hasopenrows.label("HasOpenRows"),
+                Table.hasopencolumns.label("HasOpenColumns"),
+                Table.hasopensheets.label("HasOpenSheets"),
+            )
+            .join(Table, Table.tableid == TableVersion.tableid)
+            .join(
+                TableVersionHeader, TableVersion.tablevid == TableVersionHeader.tablevid
+            )
+            .join(Header, Header.headerid == TableVersionHeader.headerid)
+            .filter(TableVersion.code.in_(table_codes))
+            .distinct()
+        )
+
+        # Apply release filter
+        query = filter_by_release(
+            query,
+            TableVersion.startreleaseid,
+            TableVersion.endreleaseid,
+            self.release_id,
+        )
+
+        # Execute query and convert to DataFrame
+        from py_dpm.models import _compile_query_for_pandas
+
+        compiled_query = _compile_query_for_pandas(query.statement, self.session)
+        df_headers = pd.read_sql(compiled_query, self.session.connection().connection)
+
         for table in table_codes:
-            table_headers = df_headers[df_headers['Code'] == table]
-            open_rows = table_headers['HasOpenRows'].values[0]
-            open_cols = table_headers['HasOpenColumns'].values[0]
-            open_sheets = table_headers['HasOpenSheets'].values[0]
+            table_headers = df_headers[df_headers["Code"] == table]
             if table_headers.empty:
                 continue
-            if "Y" in table_headers['Direction'].values and not open_rows:
-                self._check_header_present(table, 'rows')
-            if "X" in table_headers['Direction'].values and not open_cols:
-                self._check_header_present(table, 'cols')
-            if "Z" in table_headers['Direction'].values and not open_sheets:
-                self._check_header_present(table, 'sheets')
-
+            open_rows = table_headers["HasOpenRows"].values[0]
+            open_cols = table_headers["HasOpenColumns"].values[0]
+            open_sheets = table_headers["HasOpenSheets"].values[0]
+            if "Y" in table_headers["Direction"].values and not open_rows:
+                self._check_header_present(table, "rows")
+            if "X" in table_headers["Direction"].values and not open_cols:
+                self._check_header_present(table, "cols")
+            if "Z" in table_headers["Direction"].values and not open_sheets:
+                self._check_header_present(table, "sheets")
 
     def check_items(self):
         if len(self.items) == 0:
             return
         df_items = ItemCategory.get_items(self.session, self.items, self.release_id)
         if len(df_items.iloc[:, 0].values) < len(set(self.items)):
-            not_found_items = list(set(self.items).difference(set(df_items['Signature'])))
+            not_found_items = list(
+                set(self.items).difference(set(df_items["Signature"]))
+            )
             raise exceptions.SemanticError("1-1", items=not_found_items)
 
     def check_dimensions(self):
         if len(self.dimension_codes) == 0:
             return
-        self.open_keys = ViewOpenKeys.get_keys(self.session, self.dimension_codes, self.release_id)
+        self.open_keys = ViewOpenKeys.get_keys(
+            self.session, self.dimension_codes, self.release_id
+        )
         if len(self.open_keys) < len(self.dimension_codes):
-            not_found_dimensions = list(set(self.dimension_codes).difference(self.open_keys['property_code']))
+            not_found_dimensions = list(
+                set(self.dimension_codes).difference(self.open_keys["property_code"])
+            )
             raise exceptions.SemanticError("1-5", open_keys=not_found_dimensions)
 
+        # Enrich Dimension nodes with property_id from open_keys
+        # This is required by adam-engine for WHERE clause resolution
+        if self.open_keys is not None and not self.open_keys.empty:
+            # Create a mapping from dimension_code to property_id
+            property_id_map = dict(
+                zip(self.open_keys["property_code"], self.open_keys["property_id"])
+            )
+            for node in self.dimension_nodes:
+                if node.dimension_code in property_id_map:
+                    node.property_id = int(property_id_map[node.dimension_code])
+
+    def check_getop_components(self):
+        """Check and enrich GetOp nodes with property_id for their component codes.
+
+        GetOp components (like qEGS, qLGS) are property codes that need to be
+        resolved to property_id for adam-engine to process them correctly.
+        """
+        if len(self.getop_components) == 0:
+            return
+
+        # Query property_ids for GetOp components (same query as dimensions)
+        getop_keys = ViewOpenKeys.get_keys(
+            self.session, self.getop_components, self.release_id
+        )
+
+        if len(getop_keys) < len(self.getop_components):
+            not_found_components = list(
+                set(self.getop_components).difference(getop_keys["property_code"])
+            )
+            raise exceptions.SemanticError("1-5", open_keys=not_found_components)
+
+        # Enrich GetOp nodes with property_id
+        # This is required by adam-engine for [get ...] operations
+        if getop_keys is not None and not getop_keys.empty:
+            # Create a mapping from component code to property_id
+            property_id_map = dict(
+                zip(getop_keys["property_code"], getop_keys["property_id"])
+            )
+            for node in self.getop_nodes:
+                if node.component in property_id_map:
+                    node.property_id = int(property_id_map[node.component])
+
     def check_tables(self):
         for table, value in self.tables.items():
             # Extract all data and filter to get only necessary data
             table_info = value
-            df_table = ViewDatapoints.get_table_data(self.session, table, table_info['rows'],
-                                                table_info['cols'], table_info['sheets'], self.release_id)
-            if df_table.empty:
-                cell_expression = f'table: {table}'
-                for k, v in table_info.items():
-                    if v:
-                        cell_expression += f' {k}: {v}'
-                raise exceptions.SemanticError("1-2", cell_expression=cell_expression)
-
+            df_table = ViewDatapoints.get_table_data(
+                self.session,
+                table,
+                table_info["rows"],
+                table_info["cols"],
+                table_info["sheets"],
+                self.release_id,
+            )
             # Insert data type on each node by selecting only data required by node
             for node in self.operands[table]:
-                node_data = filter_all_data(df_table, table, node.rows, node.cols, node.sheets)
+                node_data = filter_all_data(
+                    df_table, table, node.rows, node.cols, node.sheets
+                )
                 # Checking grey cells (no variable ID in data for that cell)
-                grey_cells_data = node_data[node_data['variable_id'].isnull()]
+                grey_cells_data = node_data[node_data["variable_id"].isnull()]
                 if not grey_cells_data.empty:
                     if len(grey_cells_data) > 10:
                         list_cells = grey_cells_data["cell_code"].values[:10]
                     else:
                         list_cells = grey_cells_data["cell_code"].values
-                    cell_expression = ', '.join(list_cells)
-                    raise exceptions.SemanticError("1-17", cell_expression=cell_expression)
+                    cell_expression = ", ".join(list_cells)
+                    raise exceptions.SemanticError(
+                        "1-17", cell_expression=cell_expression
+                    )
                 if node_data.empty:
                     format_missing_data(node)
-                extract_data_types(node, node_data['data_type'])
-                if df_table['sheet_code'].isnull().all() and node.sheets is not None and '*' in node.sheets:
-                    raise SemanticError("1-18")
-                if df_table['row_code'].isnull().all() and node.rows is not None and '*' in node.rows:
-                    raise SemanticError("1-19")
-            del node_data
+                extract_data_types(node, node_data["data_type"])
+
+                # Check for invalid sheet wildcards
+                if (
+                    df_table["sheet_code"].isnull().all()
+                    and node.sheets is not None
+                    and "*" in node.sheets
+                ):
+                    # Check if s* is required to avoid duplicate (row, column) combinations
+                    # Group by (row_code, column_code) and check for duplicates
+                    # IMPORTANT: Include NA/NULL values in grouping (dropna=False)
+                    df_without_sheets = df_table.groupby(
+                        ["row_code", "column_code"], dropna=False
+                    ).size()
+                    has_duplicates = (df_without_sheets > 1).any()
+
+                    if not has_duplicates:
+                        # Only raise error if sheets are truly not needed (no duplicates without them)
+                        raise SemanticError("1-18")
+                    # else: s* is required even though sheet_code is NULL, so allow it
+
+                # Check for invalid row wildcards
+                if (
+                    df_table["row_code"].isnull().all()
+                    and node.rows is not None
+                    and "*" in node.rows
+                ):
+                    # Check if r* is required to avoid duplicate (column, sheet) combinations
+                    # IMPORTANT: Include NA/NULL values in grouping (dropna=False)
+                    df_without_rows = df_table.groupby(
+                        ["column_code", "sheet_code"], dropna=False
+                    ).size()
+                    has_duplicates = (df_without_rows > 1).any()
+
+                    if not has_duplicates:
+                        # Only raise error if rows are truly not needed
+                        raise SemanticError("1-19")
+                    # else: r* is required even though row_code is NULL, so allow it
+
+                # Attach node_data to the VarID node for later serialization
+                # This enables the JSON serializer to output actual cell data
+                node.data = node_data
+
+                # Expand range-type values in rows/cols/sheets to actual codes from the database
+                # This ensures adam-engine receives scalar values, not list-type ranges
+                _expand_ranges_from_data(node, node_data)
 
             # Adding data to self.data
             if self.data is None:
                 self.data = df_table
             else:
-                self.data: pd.DataFrame = pd.concat([self.data, df_table], axis=0).reset_index(drop=True)
-
+                self.data: pd.DataFrame = pd.concat(
+                    [self.data, df_table], axis=0
+                ).reset_index(drop=True)
 
-            self.key_components[table] = ViewKeyComponents.get_by_table(self.session, table, self.release_id)
+            self.key_components[table] = ViewKeyComponents.get_by_table(
+                self.session, table, self.release_id
+            )
 
     # Start of visiting nodes
     def visit_WithExpression(self, node: WithExpression):
@@ -199,7 +400,10 @@ class OperandsChecking(ASTTemplate, ABC):
 
         if self.partial_selection:
             for attribute in operand_elements:
-                if getattr(node, attribute, None) is None and not getattr(self.partial_selection, attribute, None) is None:
+                if (
+                    getattr(node, attribute, None) is None
+                    and not getattr(self.partial_selection, attribute, None) is None
+                ):
                     setattr(node, attribute, getattr(self.partial_selection, attribute))
 
         if not node.table:
@@ -208,11 +412,7 @@ class OperandsChecking(ASTTemplate, ABC):
         _create_operand_label(node)
         set_operand_label(node.label, node)
 
-        table_info = {
-            'rows': node.rows,
-            'cols': node.cols,
-            'sheets': node.sheets
-        }
+        table_info = {"rows": node.rows, "cols": node.cols, "sheets": node.sheets}
 
         if node.table not in self.tables:
             self.tables[node.table] = table_info
@@ -224,17 +424,32 @@ class OperandsChecking(ASTTemplate, ABC):
     def visit_Dimension(self, node: Dimension):
         if node.dimension_code not in self.dimension_codes:
             self.dimension_codes.append(node.dimension_code)
+        # Store reference to node for property_id enrichment
+        self.dimension_nodes.append(node)
+
+    def visit_GetOp(self, node: GetOp):
+        """Visit GetOp nodes to collect component codes for property_id lookup."""
+        if node.component not in self.getop_components:
+            self.getop_components.append(node.component)
+        # Store reference to node for property_id enrichment
+        self.getop_nodes.append(node)
+        # Visit the operand to ensure it gets processed (e.g., VarID nodes inside GetOp)
+        self.visit(node.operand)
 
     def visit_VarRef(self, node: VarRef):
-        if not VariableVersion.check_variable_exists(self.session, node.variable, self.release_id):
-            raise exceptions.SemanticError('1-3', variable=node.variable)
+        if not VariableVersion.check_variable_exists(
+            self.session, node.variable, self.release_id
+        ):
+            raise exceptions.SemanticError("1-3", variable=node.variable)
 
     def visit_PreconditionItem(self, node: PreconditionItem):
 
         if self.is_scripting:
-            raise exceptions.SemanticError('6-3', precondition=node.variable_code)
+            raise exceptions.SemanticError("6-3", precondition=node.variable_code)
 
-        if not VariableVersion.check_variable_exists(self.session, node.variable_code, self.release_id):
+        if not VariableVersion.check_variable_exists(
+            self.session, node.variable_code, self.release_id
+        ):
             raise exceptions.SemanticError("1-3", variable=node.variable_code)
 
         self.preconditions = True
@@ -242,7 +457,7 @@ class OperandsChecking(ASTTemplate, ABC):
         set_operand_label(node.label, node)
 
     def visit_Scalar(self, node: Scalar):
-        if node.item and node.scalar_type == 'Item':
+        if node.item and node.scalar_type == "Item":
             if node.item not in self.items:
                 self.items.append(node.item)
 
@@ -268,11 +483,16 @@ class OperandsChecking(ASTTemplate, ABC):
 
     def check_operations(self):
         if len(self.operations):
-            df_operations = Operation.get_operations_from_codes(session=self.session, operation_codes=self.operations,
-                                                                release_id=self.release_id)
+            df_operations = Operation.get_operations_from_codes(
+                session=self.session,
+                operation_codes=self.operations,
+                release_id=self.release_id,
+            )
             if len(df_operations.values) < len(self.operations):
-                not_found_operations = list(set(self.operations).difference(set(df_operations['Code'])))
-                raise exceptions.SemanticError('1-8', operations=not_found_operations)
+                not_found_operations = list(
+                    set(self.operations).difference(set(df_operations["Code"]))
+                )
+                raise exceptions.SemanticError("1-8", operations=not_found_operations)
             self.operations_data = df_operations
 
 
@@ -292,7 +512,10 @@ def extract_data_types(node: VarID, database_types: pd.Series) -> None:
         else:
             setattr(node, "type", data_type())
     else:
-        data_types = {scalar_factory.database_types_mapping(data_type) for data_type in unique_types}
+        data_types = {
+            scalar_factory.database_types_mapping(data_type)
+            for data_type in unique_types
+        }
         if len(data_types) == 1:
             data_type = data_types.pop()
             setattr(node, "type", data_type())