pydpm_xl 0.2.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

py_dpm/__init__.py

@@ -41,7 +41,7 @@ Available packages:
 - pydpm.api: Main APIs for migration, syntax, and semantic analysis
 """
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 __author__ = "MeaningfulData S.L."
 __email__ = "info@meaningfuldata.eu"
 __license__ = "GPL-3.0-or-later"

py_dpm/api/dpm_xl/complete_ast.py

@@ -10,7 +10,7 @@ Also provides enrichment functionality to create engine-ready ASTs with framewor
 """
 
 from datetime import datetime
-from typing import Dict, Any, Optional
+from typing import Dict, Any, Any, Optional
 from py_dpm.dpm_xl.utils.serialization import ASTToJSONVisitor
 
 
@@ -23,8 +23,8 @@ def generate_complete_ast(
     """
     Generate complete AST with all data fields, exactly like json_scripts examples.
 
-    This function replicates the exact same process used to generate the reference
-    JSON files in json_scripts/, ensuring complete data field population.
+    This function replicates the process used to generate the reference JSON files,
+    using the new SemanticAPI to perform full semantic validation and operand checking.
 
     Args:
         expression: DPM-XL expression string
@@ -34,25 +34,18 @@ def generate_complete_ast(
             If None, uses all available data (release-agnostic).
 
     Returns:
-        dict: {
-            'success': bool,
-            'ast': dict,        # Complete AST with data fields
-            'context': dict,    # Context from WITH clause
-            'error': str,       # Error if failed
-            'data_populated': bool  # Whether data fields were populated
-        }
+        dict with keys:
+            success, ast, context, error, data_populated, semantic_result
     """
     try:
         # Import here to avoid circular imports
-        from py_dpm.api import API
+        from py_dpm.api.dpm_xl.semantic import SemanticAPI
         from py_dpm.dpm.utils import get_engine
 
-        # Initialize database connection if provided
+        # Initialize database connection if explicitly provided, to surface connection errors early
         if connection_url or database_path:
             try:
-                engine = get_engine(
-                    database_path=database_path, connection_url=connection_url
-                )
+                get_engine(database_path=database_path, connection_url=connection_url)
             except Exception as e:
                 return {
                     "success": False,
@@ -62,194 +55,79 @@ def generate_complete_ast(
                     "data_populated": False,
                 }
 
-        # Use the legacy API which does complete semantic validation
-        # This is the same API used to generate the original JSON files
-        api = API(database_path=database_path, connection_url=connection_url)
-
-        # Perform complete semantic validation with operand checking
-        # This should populate all data fields on VarID nodes
-        semantic_result = api.semantic_validation(expression, release_id=release_id)
-
-        # Force data population if semantic validation completed successfully
-        if hasattr(api, "AST") and api.AST and semantic_result:
-            try:
-                from py_dpm.dpm_xl.ast.operands import OperandsChecking
-                from py_dpm.dpm.utils import get_session
-
-                session = get_session()
-
-                # Extract the expression AST
-                def get_inner_ast(ast_obj):
-                    if hasattr(ast_obj, "children") and len(ast_obj.children) > 0:
-                        child = ast_obj.children[0]
-                        if hasattr(child, "expression"):
-                            return child.expression
-                        else:
-                            return child
-                    return ast_obj
-
-                inner_ast = get_inner_ast(api.AST)
-
-                # Run operand checking to populate data fields
-                oc = OperandsChecking(
-                    session=session,
-                    expression=expression,
-                    ast=inner_ast,
-                    release_id=release_id,
-                )
-
-                # Apply the data from operand checker to VarID nodes
-                if hasattr(oc, "data") and oc.data is not None:
-
-                    # Apply data to VarID nodes in the AST
-                    def apply_data_to_varids(node):
-                        if (
-                            hasattr(node, "__class__")
-                            and node.__class__.__name__ == "VarID"
-                        ):
-                            table = getattr(node, "table", None)
-                            rows = getattr(node, "rows", None)
-                            cols = getattr(node, "cols", None)
-
-                            if table and table in oc.operands:
-                                # Filter data for this specific VarID
-                                # Start with table filter
-                                filter_mask = oc.data["table_code"] == table
-
-                                # Add row filter only if rows is not None and doesn't contain wildcards
-                                # IMPORTANT: If rows contains '*', include all rows (don't filter)
-                                if rows is not None and "*" not in rows:
-                                    filter_mask = filter_mask & (
-                                        oc.data["row_code"].isin(rows)
-                                    )
-
-                                # Add column filter only if cols is not None and doesn't contain wildcards
-                                # IMPORTANT: If cols contains '*', include all columns (don't filter)
-                                if cols is not None and "*" not in cols:
-                                    filter_mask = filter_mask & (
-                                        oc.data["column_code"].isin(cols)
-                                    )
-
-                                filtered_data = oc.data[filter_mask]
-
-                                if not filtered_data.empty:
-                                    # IMPORTANT: Remove wildcard entries (NULL column/row/sheet codes)
-                                    # when specific entries exist for the same dimension
-                                    # The database contains both wildcard entries (column_code=NULL for c*)
-                                    # and specific entries (column_code='0010'). When we query with wildcards,
-                                    # we want only the specific entries.
-
-                                    # Remove rows where column_code is NULL if there are non-NULL column_code entries
-                                    if filtered_data["column_code"].notna().any():
-                                        filtered_data = filtered_data[
-                                            filtered_data["column_code"].notna()
-                                        ]
-
-                                    # Remove rows where row_code is NULL if there are non-NULL row_code entries
-                                    if filtered_data["row_code"].notna().any():
-                                        filtered_data = filtered_data[
-                                            filtered_data["row_code"].notna()
-                                        ]
-
-                                    # Remove rows where sheet_code is NULL if there are non-NULL sheet_code entries
-                                    if filtered_data["sheet_code"].notna().any():
-                                        filtered_data = filtered_data[
-                                            filtered_data["sheet_code"].notna()
-                                        ]
-
-                                    # IMPORTANT: After filtering, remove any remaining duplicates
-                                    # based on (row_code, column_code, sheet_code) combination
-                                    filtered_data = filtered_data.drop_duplicates(
-                                        subset=[
-                                            "row_code",
-                                            "column_code",
-                                            "sheet_code",
-                                        ],
-                                        keep="first",
-                                    )
-
-                                    # Set the data attribute on the VarID node
-                                    if not filtered_data.empty:
-                                        node.data = filtered_data
-
-                        # Recursively apply to child nodes
-                        for attr_name in [
-                            "children",
-                            "left",
-                            "right",
-                            "operand",
-                            "operands",
-                            "expression",
-                            "condition",
-                            "then_expr",
-                            "else_expr",
-                        ]:
-                            if hasattr(node, attr_name):
-                                attr_value = getattr(node, attr_name)
-                                if isinstance(attr_value, list):
-                                    for item in attr_value:
-                                        if hasattr(item, "__class__"):
-                                            apply_data_to_varids(item)
-                                elif attr_value and hasattr(attr_value, "__class__"):
-                                    apply_data_to_varids(attr_value)
-
-                    # Apply data to all VarID nodes in the AST
-                    apply_data_to_varids(inner_ast)
+        # Use the modern SemanticAPI which performs full semantic validation and operand checking
+        semantic_api = SemanticAPI(
+            database_path=database_path, connection_url=connection_url
+        )
 
-            except Exception as e:
-                # Silently continue if data population fails
-                pass
-
-        if hasattr(api, "AST") and api.AST is not None:
-            # Extract components exactly like batch_validator does
-            def extract_components(ast_obj):
-                if hasattr(ast_obj, "children") and len(ast_obj.children) > 0:
-                    child = ast_obj.children[0]
-                    if hasattr(child, "expression"):
-                        return child.expression, child.partial_selection
-                    else:
-                        return child, None
-                return ast_obj, None
-
-            actual_ast, context = extract_components(api.AST)
-
-            # Convert to JSON exactly like batch_validator does
-            visitor = ASTToJSONVisitor(context)
-            ast_dict = visitor.visit(actual_ast)
-
-            # Check if data fields were populated
-            data_populated = _check_data_fields_populated(ast_dict)
-
-            # Serialize context
-            context_dict = None
-            if context:
-                context_dict = {
-                    "table": getattr(context, "table", None),
-                    "rows": getattr(context, "rows", None),
-                    "columns": getattr(context, "cols", None),
-                    "sheets": getattr(context, "sheets", None),
-                    "default": getattr(context, "default", None),
-                    "interval": getattr(context, "interval", None),
-                }
+        semantic_result = semantic_api.validate_expression(
+            expression, release_id=release_id
+        )
 
+        # If semantic validation failed, return structured error
+        if not semantic_result.is_valid:
             return {
-                "success": True,
-                "ast": ast_dict,
-                "context": context_dict,
-                "error": None,
-                "data_populated": data_populated,
+                "success": False,
+                "ast": None,
+                "context": None,
+                "error": semantic_result.error_message,
+                "data_populated": False,
                 "semantic_result": semantic_result,
             }
 
-        else:
+        ast_root = getattr(semantic_api, "ast", None)
+
+        if ast_root is None:
             return {
                 "success": False,
                 "ast": None,
                 "context": None,
                 "error": "Semantic validation did not generate AST",
                 "data_populated": False,
+                "semantic_result": semantic_result,
+            }
+
+        # Extract components exactly like batch_validator does
+        def extract_components(ast_obj):
+            if hasattr(ast_obj, "children") and len(ast_obj.children) > 0:
+                child = ast_obj.children[0]
+                if hasattr(child, "expression"):
+                    return child.expression, child.partial_selection
+                else:
+                    return child, None
+            return ast_obj, None
+
+        actual_ast, context = extract_components(ast_root)
+
+        # Convert to JSON using the ASTToJSONVisitor, which uses VarID.data populated
+        # during semantic validation / operand checking.
+        visitor = ASTToJSONVisitor(context)
+        ast_dict = visitor.visit(actual_ast)
+
+        # Check if data fields were populated
+        data_populated = _check_data_fields_populated(ast_dict)
+
+        # Serialize context
+        context_dict = None
+        if context:
+            context_dict = {
+                "table": getattr(context, "table", None),
+                "rows": getattr(context, "rows", None),
+                "columns": getattr(context, "cols", None),
+                "sheets": getattr(context, "sheets", None),
+                "default": getattr(context, "default", None),
+                "interval": getattr(context, "interval", None),
             }
 
+        return {
+            "success": True,
+            "ast": ast_dict,
+            "context": context_dict,
+            "error": None,
+            "data_populated": data_populated,
+            "semantic_result": semantic_result,
+        }
+
     except Exception as e:
         return {
             "success": False,

py_dpm/api/dpm_xl/semantic.py

@@ -56,6 +56,8 @@ class SemanticAPI:
         """
         self.database_path = database_path
         self.connection_url = connection_url
+        # Store last parsed AST for consumers that need it (e.g. complete AST generation)
+        self.ast = None
 
         if connection_url:
             # Create isolated engine and session for the provided connection URL
@@ -141,6 +143,8 @@ class SemanticAPI:
 
             # Generate AST
             ast = self.visitor.visit(parse_tree)
+            # Expose AST on the instance for downstream consumers
+            self.ast = ast
 
             # Perform semantic analysis
             oc = OperandsChecking(

py_dpm/cli/main.py

@@ -6,7 +6,7 @@ import os
 import sys
 import pandas as pd
 
-from py_dpm.api import SemanticAPI
+from py_dpm.api import SemanticAPI, SyntaxAPI
 from py_dpm.api.dpm_xl.semantic import SemanticValidationResult
 from py_dpm.api.dpm.operation_scopes import OperationScopesAPI
 from py_dpm.dpm.migration import run_migration
@@ -147,9 +147,11 @@ def syntax(expression: str):
     """Perform syntactic analysis on a DPM expression."""
 
     status = 0
-    api = API()
+    api = SyntaxAPI()
     try:
-        api.syntax_validation(expression)
+        result = api.validate_expression(expression)
+        if not result.is_valid:
+            raise SyntaxError(result.error_message or "Syntax errors detected")
         message_formatted = Text("Syntax OK", style="bold green")
     except SyntaxError as e:
         message = str(e)

py_dpm/dpm_xl/utils/serialization.py

@@ -542,10 +542,9 @@ class ASTToJSONVisitor(NodeVisitor):
 
         return result
 
-
 # Original serialization functions (kept for backward compatibility)
 import json
-from py_dpm.dpm_xl.ast import nodes
+from py_dpm.dpm_xl.ast import nodes as ASTObjects
 
 
 def expand_with_expression(node):
@@ -759,4 +758,4 @@ def ast_from_json_string(json_str):
         AST object instance
     """
     data = json.loads(json_str)
-    return deserialize_ast(data)
+    return deserialize_ast(data)

{pydpm_xl-0.2.0.dist-info → pydpm_xl-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydpm_xl
-Version: 0.2.0
+Version: 0.2.1
 Summary: Python library for DPM-XL data processing and analysis
 Author-email: "MeaningfulData S.L." <info@meaningfuldata.eu>
 License: GPL-3.0-or-later

{pydpm_xl-0.2.0.dist-info → pydpm_xl-0.2.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-py_dpm/__init__.py,sha256=rbhfed7zNfD34iEi3bPOB9D7x1mpkcUaXZjtqQecZfo,1858
+py_dpm/__init__.py,sha256=SfCnl2zNEvxu7CWIE2i9T-IlPrYBwynS4qSBFRloO2E,1858
 py_dpm/api/__init__.py,sha256=g0w2FOtETU1ZMuhGnbdgBmmTZwmWoBtBEDmkLR1t8sc,1824
 py_dpm/api/explorer.py,sha256=1dBc2ZidcAR5DfPEBRiyVk54Xv9u_G9mU66u7PA9Z-E,86
 py_dpm/api/semantic.py,sha256=QT0znXXa4ihqk4GmlmJQrdceCRy1o_3J17koW6rT5PE,1304
@@ -10,11 +10,11 @@ py_dpm/api/dpm/migration.py,sha256=9FT7zzz4QdUIRR6MD01gMODBtfq9HH_RF4hRgZqMcZc,2
 py_dpm/api/dpm/operation_scopes.py,sha256=Vw9cdPiFM7uCsQyMY1AgT5XhHbxFcVeMwIcIXANuty4,48486
 py_dpm/api/dpm_xl/__init__.py,sha256=rjiIf9XDi2IGf0G_LiOWp29e5ANyoREfzl5Z5phJU_8,603
 py_dpm/api/dpm_xl/ast_generator.py,sha256=-wOgUEM1DMpyVwflkplLr7BOZFfjaDeXi-R_PLhsAxo,16160
-py_dpm/api/dpm_xl/complete_ast.py,sha256=rBfzGQ5RnJXlLcG46jMpArfnllQSPN1ny4Ou_Y_HGAc,31002
-py_dpm/api/dpm_xl/semantic.py,sha256=hbyS0tQITZjeXNid6j4Rvc_PkV-8tfmZrj2_1XlAKwg,13422
+py_dpm/api/dpm_xl/complete_ast.py,sha256=XmXvE1zWLxL732FvyUpK4WffKwZSrw5yrzJI-QGOpFI,24341
+py_dpm/api/dpm_xl/semantic.py,sha256=Buo_t-sEv65r6RmYDy1xkCWGlU2pB2WQsDM-X-FX4cc,13629
 py_dpm/api/dpm_xl/syntax.py,sha256=Ke_kKd9ModoJ6siL3GPT9j9QClmopryCRcdDAT3M5-E,5954
 py_dpm/cli/__init__.py,sha256=UrfGHoQ0sZLjWfA0hoOoI4iTrn-bjr2f9Q8wDWd5nMo,133
-py_dpm/cli/main.py,sha256=UACXaWiPZH6vxfmdasSMTVXAk7IceOcon30qtOugPz4,22304
+py_dpm/cli/main.py,sha256=Cu04d3hpv0UdcROdkvpNipNfiAkde59YCbPu0gKKCYA,22444
 py_dpm/cli/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 py_dpm/dpm/__init__.py,sha256=moagUo5Gxf24-Tl9FL_3n2wmVoD_oXtpC-YIGktH_rc,212
 py_dpm/dpm/migration.py,sha256=ivO_ObvKzVomTns6qfo-o5FuciWxkXbMd_gJ4_tu7Xc,14110
@@ -70,7 +70,7 @@ py_dpm/dpm_xl/utils/data_handlers.py,sha256=a0E-IaP_-CDKLcj-Gt2ggAziKIOUiwnT2D9I
 py_dpm/dpm_xl/utils/operands_mapping.py,sha256=LG0hPlUuTM2X2uWOtiD6HkmNeDEJkWJ8gV-Fxej_8QM,2241
 py_dpm/dpm_xl/utils/operator_mapping.py,sha256=BFgbVbSCSuutFNHJ4gtgm5VuG38pcl8Kmfi-sefg6JU,1913
 py_dpm/dpm_xl/utils/scopes_calculator.py,sha256=nCx2mz_qtw61BESp38ORQYlF2uRT8SyUKawSX9OQljM,17832
-py_dpm/dpm_xl/utils/serialization.py,sha256=VkjTTcg7okQ9uw1WGsM9xyYGRTrHcDjEuveeS92taxM,32468
+py_dpm/dpm_xl/utils/serialization.py,sha256=LPcmudFfzHeEjIIr57kr5BvGPZbxshOAAeUYOrLl7XM,32482
 py_dpm/dpm_xl/utils/tokens.py,sha256=VRIrPDi5ttwgH-on5Qt4-l4ho4bLA755-nfTalponcA,3496
 py_dpm/dpm_xl/validation/__init__.py,sha256=pzSTkLzUgK16XsV_maXn__erzs0blDBDRApuRWJwwMs,250
 py_dpm/dpm_xl/validation/generation_utils.py,sha256=2UvYorpLor-8PiEUTWFFVr-xEzKjxQzZrLXz3lvedPk,22655
@@ -80,9 +80,9 @@ py_dpm/dpm_xl/validation/variants.py,sha256=LM_2U_sCf8rnlUyt6k4WQwSKj5eTJLt4Ba8q
 py_dpm/exceptions/__init__.py,sha256=yDERfUxYW7NUUEiTQChGpuJx6abr7IDe2XUpwVFPtvM,416
 py_dpm/exceptions/exceptions.py,sha256=6S3p-_i5O1oStvSMixt_JQG0xwTeSfBcdzrwL8yBy6Q,2413
 py_dpm/exceptions/messages.py,sha256=UwY6QIK8c-POcDCc9HYbZFGArCIYAanUGNh2LNKPx3U,7534
-pydpm_xl-0.2.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-pydpm_xl-0.2.0.dist-info/METADATA,sha256=O0-W3yLmeS50GBcIt4VBH0cB4tEMApbJS7Vnjxr4u00,7974
-pydpm_xl-0.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pydpm_xl-0.2.0.dist-info/entry_points.txt,sha256=6DDmBfw-AjtgvMHgq_I730i_LAAs_7-N3C95HD_bRr4,47
-pydpm_xl-0.2.0.dist-info/top_level.txt,sha256=495PvWZRoKl2NvbQU25W7dqWIBHqY-mFMPt83uxPpcM,7
-pydpm_xl-0.2.0.dist-info/RECORD,,
+pydpm_xl-0.2.1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+pydpm_xl-0.2.1.dist-info/METADATA,sha256=rUtdLeAJq0dbweFf9b5YXGC2wTPSU73pGuCgUyaek4Y,7974
+pydpm_xl-0.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pydpm_xl-0.2.1.dist-info/entry_points.txt,sha256=6DDmBfw-AjtgvMHgq_I730i_LAAs_7-N3C95HD_bRr4,47
+pydpm_xl-0.2.1.dist-info/top_level.txt,sha256=495PvWZRoKl2NvbQU25W7dqWIBHqY-mFMPt83uxPpcM,7
+pydpm_xl-0.2.1.dist-info/RECORD,,