pydpm_xl 0.2.0__tar.gz → 0.2.2__tar.gz

{pydpm_xl-0.2.0/pydpm_xl.egg-info → pydpm_xl-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydpm_xl
-Version: 0.2.0
+Version: 0.2.2
 Summary: Python library for DPM-XL data processing and analysis
 Author-email: "MeaningfulData S.L." <info@meaningfuldata.eu>
 License: GPL-3.0-or-later

{pydpm_xl-0.2.0 → pydpm_xl-0.2.2}/py_dpm/__init__.py

@@ -41,7 +41,7 @@ Available packages:
 - pydpm.api: Main APIs for migration, syntax, and semantic analysis
 """
 
-__version__ = "0.2.0"
+__version__ = "0.2.2"
 __author__ = "MeaningfulData S.L."
 __email__ = "info@meaningfuldata.eu"
 __license__ = "GPL-3.0-or-later"

{pydpm_xl-0.2.0 → pydpm_xl-0.2.2}/py_dpm/api/dpm_xl/complete_ast.py

@@ -10,7 +10,7 @@ Also provides enrichment functionality to create engine-ready ASTs with framewor
 """
 
 from datetime import datetime
-from typing import Dict, Any, Optional
+from typing import Dict, Any, Any, Optional
 from py_dpm.dpm_xl.utils.serialization import ASTToJSONVisitor
 
 
@@ -23,8 +23,8 @@ def generate_complete_ast(
     """
     Generate complete AST with all data fields, exactly like json_scripts examples.
 
-    This function replicates the exact same process used to generate the reference
-    JSON files in json_scripts/, ensuring complete data field population.
+    This function replicates the process used to generate the reference JSON files,
+    using the new SemanticAPI to perform full semantic validation and operand checking.
 
     Args:
         expression: DPM-XL expression string
@@ -34,25 +34,18 @@ def generate_complete_ast(
             If None, uses all available data (release-agnostic).
 
     Returns:
-        dict: {
-            'success': bool,
-            'ast': dict,        # Complete AST with data fields
-            'context': dict,    # Context from WITH clause
-            'error': str,       # Error if failed
-            'data_populated': bool  # Whether data fields were populated
-        }
+        dict with keys:
+            success, ast, context, error, data_populated, semantic_result
     """
     try:
         # Import here to avoid circular imports
-        from py_dpm.api import API
+        from py_dpm.api.dpm_xl.semantic import SemanticAPI
         from py_dpm.dpm.utils import get_engine
 
-        # Initialize database connection if provided
+        # Initialize database connection if explicitly provided, to surface connection errors early
         if connection_url or database_path:
             try:
-                engine = get_engine(
-                    database_path=database_path, connection_url=connection_url
-                )
+                get_engine(database_path=database_path, connection_url=connection_url)
             except Exception as e:
                 return {
                     "success": False,
@@ -62,194 +55,79 @@ def generate_complete_ast(
                     "data_populated": False,
                 }
 
-        # Use the legacy API which does complete semantic validation
-        # This is the same API used to generate the original JSON files
-        api = API(database_path=database_path, connection_url=connection_url)
-
-        # Perform complete semantic validation with operand checking
-        # This should populate all data fields on VarID nodes
-        semantic_result = api.semantic_validation(expression, release_id=release_id)
-
-        # Force data population if semantic validation completed successfully
-        if hasattr(api, "AST") and api.AST and semantic_result:
-            try:
-                from py_dpm.dpm_xl.ast.operands import OperandsChecking
-                from py_dpm.dpm.utils import get_session
-
-                session = get_session()
-
-                # Extract the expression AST
-                def get_inner_ast(ast_obj):
-                    if hasattr(ast_obj, "children") and len(ast_obj.children) > 0:
-                        child = ast_obj.children[0]
-                        if hasattr(child, "expression"):
-                            return child.expression
-                        else:
-                            return child
-                    return ast_obj
-
-                inner_ast = get_inner_ast(api.AST)
-
-                # Run operand checking to populate data fields
-                oc = OperandsChecking(
-                    session=session,
-                    expression=expression,
-                    ast=inner_ast,
-                    release_id=release_id,
-                )
-
-                # Apply the data from operand checker to VarID nodes
-                if hasattr(oc, "data") and oc.data is not None:
-
-                    # Apply data to VarID nodes in the AST
-                    def apply_data_to_varids(node):
-                        if (
-                            hasattr(node, "__class__")
-                            and node.__class__.__name__ == "VarID"
-                        ):
-                            table = getattr(node, "table", None)
-                            rows = getattr(node, "rows", None)
-                            cols = getattr(node, "cols", None)
-
-                            if table and table in oc.operands:
-                                # Filter data for this specific VarID
-                                # Start with table filter
-                                filter_mask = oc.data["table_code"] == table
-
-                                # Add row filter only if rows is not None and doesn't contain wildcards
-                                # IMPORTANT: If rows contains '*', include all rows (don't filter)
-                                if rows is not None and "*" not in rows:
-                                    filter_mask = filter_mask & (
-                                        oc.data["row_code"].isin(rows)
-                                    )
-
-                                # Add column filter only if cols is not None and doesn't contain wildcards
-                                # IMPORTANT: If cols contains '*', include all columns (don't filter)
-                                if cols is not None and "*" not in cols:
-                                    filter_mask = filter_mask & (
-                                        oc.data["column_code"].isin(cols)
-                                    )
-
-                                filtered_data = oc.data[filter_mask]
-
-                                if not filtered_data.empty:
-                                    # IMPORTANT: Remove wildcard entries (NULL column/row/sheet codes)
-                                    # when specific entries exist for the same dimension
-                                    # The database contains both wildcard entries (column_code=NULL for c*)
-                                    # and specific entries (column_code='0010'). When we query with wildcards,
-                                    # we want only the specific entries.
-
-                                    # Remove rows where column_code is NULL if there are non-NULL column_code entries
-                                    if filtered_data["column_code"].notna().any():
-                                        filtered_data = filtered_data[
-                                            filtered_data["column_code"].notna()
-                                        ]
-
-                                    # Remove rows where row_code is NULL if there are non-NULL row_code entries
-                                    if filtered_data["row_code"].notna().any():
-                                        filtered_data = filtered_data[
-                                            filtered_data["row_code"].notna()
-                                        ]
-
-                                    # Remove rows where sheet_code is NULL if there are non-NULL sheet_code entries
-                                    if filtered_data["sheet_code"].notna().any():
-                                        filtered_data = filtered_data[
-                                            filtered_data["sheet_code"].notna()
-                                        ]
-
-                                    # IMPORTANT: After filtering, remove any remaining duplicates
-                                    # based on (row_code, column_code, sheet_code) combination
-                                    filtered_data = filtered_data.drop_duplicates(
-                                        subset=[
-                                            "row_code",
-                                            "column_code",
-                                            "sheet_code",
-                                        ],
-                                        keep="first",
-                                    )
-
-                                    # Set the data attribute on the VarID node
-                                    if not filtered_data.empty:
-                                        node.data = filtered_data
-
-                        # Recursively apply to child nodes
-                        for attr_name in [
-                            "children",
-                            "left",
-                            "right",
-                            "operand",
-                            "operands",
-                            "expression",
-                            "condition",
-                            "then_expr",
-                            "else_expr",
-                        ]:
-                            if hasattr(node, attr_name):
-                                attr_value = getattr(node, attr_name)
-                                if isinstance(attr_value, list):
-                                    for item in attr_value:
-                                        if hasattr(item, "__class__"):
-                                            apply_data_to_varids(item)
-                                elif attr_value and hasattr(attr_value, "__class__"):
-                                    apply_data_to_varids(attr_value)
-
-                    # Apply data to all VarID nodes in the AST
-                    apply_data_to_varids(inner_ast)
+        # Use the modern SemanticAPI which performs full semantic validation and operand checking
+        semantic_api = SemanticAPI(
+            database_path=database_path, connection_url=connection_url
+        )
 
-            except Exception as e:
-                # Silently continue if data population fails
-                pass
-
-        if hasattr(api, "AST") and api.AST is not None:
-            # Extract components exactly like batch_validator does
-            def extract_components(ast_obj):
-                if hasattr(ast_obj, "children") and len(ast_obj.children) > 0:
-                    child = ast_obj.children[0]
-                    if hasattr(child, "expression"):
-                        return child.expression, child.partial_selection
-                    else:
-                        return child, None
-                return ast_obj, None
-
-            actual_ast, context = extract_components(api.AST)
-
-            # Convert to JSON exactly like batch_validator does
-            visitor = ASTToJSONVisitor(context)
-            ast_dict = visitor.visit(actual_ast)
-
-            # Check if data fields were populated
-            data_populated = _check_data_fields_populated(ast_dict)
-
-            # Serialize context
-            context_dict = None
-            if context:
-                context_dict = {
-                    "table": getattr(context, "table", None),
-                    "rows": getattr(context, "rows", None),
-                    "columns": getattr(context, "cols", None),
-                    "sheets": getattr(context, "sheets", None),
-                    "default": getattr(context, "default", None),
-                    "interval": getattr(context, "interval", None),
-                }
+        semantic_result = semantic_api.validate_expression(
+            expression, release_id=release_id
+        )
 
+        # If semantic validation failed, return structured error
+        if not semantic_result.is_valid:
             return {
-                "success": True,
-                "ast": ast_dict,
-                "context": context_dict,
-                "error": None,
-                "data_populated": data_populated,
+                "success": False,
+                "ast": None,
+                "context": None,
+                "error": semantic_result.error_message,
+                "data_populated": False,
                 "semantic_result": semantic_result,
             }
 
-        else:
+        ast_root = getattr(semantic_api, "ast", None)
+
+        if ast_root is None:
             return {
                 "success": False,
                 "ast": None,
                 "context": None,
                 "error": "Semantic validation did not generate AST",
                 "data_populated": False,
+                "semantic_result": semantic_result,
+            }
+
+        # Extract components exactly like batch_validator does
+        def extract_components(ast_obj):
+            if hasattr(ast_obj, "children") and len(ast_obj.children) > 0:
+                child = ast_obj.children[0]
+                if hasattr(child, "expression"):
+                    return child.expression, child.partial_selection
+                else:
+                    return child, None
+            return ast_obj, None
+
+        actual_ast, context = extract_components(ast_root)
+
+        # Convert to JSON using the ASTToJSONVisitor, which uses VarID.data populated
+        # during semantic validation / operand checking.
+        visitor = ASTToJSONVisitor(context)
+        ast_dict = visitor.visit(actual_ast)
+
+        # Check if data fields were populated
+        data_populated = _check_data_fields_populated(ast_dict)
+
+        # Serialize context
+        context_dict = None
+        if context:
+            context_dict = {
+                "table": getattr(context, "table", None),
+                "rows": getattr(context, "rows", None),
+                "columns": getattr(context, "cols", None),
+                "sheets": getattr(context, "sheets", None),
+                "default": getattr(context, "default", None),
+                "interval": getattr(context, "interval", None),
             }
 
+        return {
+            "success": True,
+            "ast": ast_dict,
+            "context": context_dict,
+            "error": None,
+            "data_populated": data_populated,
+            "semantic_result": semantic_result,
+        }
+
     except Exception as e:
         return {
             "success": False,

{pydpm_xl-0.2.0 → pydpm_xl-0.2.2}/py_dpm/api/dpm_xl/semantic.py

@@ -56,6 +56,8 @@ class SemanticAPI:
         """
         self.database_path = database_path
         self.connection_url = connection_url
+        # Store last parsed AST for consumers that need it (e.g. complete AST generation)
+        self.ast = None
 
         if connection_url:
             # Create isolated engine and session for the provided connection URL
@@ -141,6 +143,8 @@ class SemanticAPI:
 
             # Generate AST
             ast = self.visitor.visit(parse_tree)
+            # Expose AST on the instance for downstream consumers
+            self.ast = ast
 
             # Perform semantic analysis
             oc = OperandsChecking(

{pydpm_xl-0.2.0 → pydpm_xl-0.2.2}/py_dpm/cli/main.py

@@ -6,7 +6,7 @@ import os
 import sys
 import pandas as pd
 
-from py_dpm.api import SemanticAPI
+from py_dpm.api import SemanticAPI, SyntaxAPI
 from py_dpm.api.dpm_xl.semantic import SemanticValidationResult
 from py_dpm.api.dpm.operation_scopes import OperationScopesAPI
 from py_dpm.dpm.migration import run_migration
@@ -147,9 +147,11 @@ def syntax(expression: str):
     """Perform syntactic analysis on a DPM expression."""
 
     status = 0
-    api = API()
+    api = SyntaxAPI()
     try:
-        api.syntax_validation(expression)
+        result = api.validate_expression(expression)
+        if not result.is_valid:
+            raise SyntaxError(result.error_message or "Syntax errors detected")
         message_formatted = Text("Syntax OK", style="bold green")
     except SyntaxError as e:
         message = str(e)

{pydpm_xl-0.2.0 → pydpm_xl-0.2.2}/py_dpm/dpm/queries/tables.py

@@ -74,7 +74,12 @@ class TableQuery:
             subq.c.table_code.isnot(None)
         )
 
-        q = filter_by_release(q, release_id, subq.c.start_release, subq.c.end_release)
+        q = filter_by_release(
+            q,
+            start_col=subq.c.start_release,
+            end_col=subq.c.end_release,
+            release_id=release_id,
+        )
         q = q.order_by(subq.c.table_code)
 
         return BaseQuery(session, q)
@@ -91,7 +96,12 @@ class TableQuery:
             subq.c.table_code == table_code, subq.c.row_code.isnot(None)
         )
 
-        q = filter_by_release(q, release_id, subq.c.start_release, subq.c.end_release)
+        q = filter_by_release(
+            q,
+            start_col=subq.c.start_release,
+            end_col=subq.c.end_release,
+            release_id=release_id,
+        )
         q = q.order_by(subq.c.row_code)
 
         return BaseQuery(session, q)
@@ -108,7 +118,12 @@ class TableQuery:
             subq.c.table_code == table_code, subq.c.column_code.isnot(None)
         )
 
-        q = filter_by_release(q, release_id, subq.c.start_release, subq.c.end_release)
+        q = filter_by_release(
+            q,
+            start_col=subq.c.start_release,
+            end_col=subq.c.end_release,
+            release_id=release_id,
+        )
         q = q.order_by(subq.c.column_code)
 
         return BaseQuery(session, q)
@@ -127,7 +142,12 @@ class TableQuery:
             subq.c.sheet_code != "",
         )
 
-        q = filter_by_release(q, release_id, subq.c.start_release, subq.c.end_release)
+        q = filter_by_release(
+            q,
+            start_col=subq.c.start_release,
+            end_col=subq.c.end_release,
+            release_id=release_id,
+        )
         q = q.order_by(subq.c.sheet_code)
 
         return BaseQuery(session, q)

{pydpm_xl-0.2.0 → pydpm_xl-0.2.2}/py_dpm/dpm_xl/ast/operands.py

@@ -208,9 +208,9 @@ class OperandsChecking(ASTTemplate, ABC):
         # Apply release filter
         query = filter_by_release(
             query,
-            self.release_id,
-            TableVersion.startreleaseid,
-            TableVersion.endreleaseid,
+            start_col=TableVersion.startreleaseid,
+            end_col=TableVersion.endreleaseid,
+            release_id=self.release_id,
         )
 
         # Execute query and convert to DataFrame

{pydpm_xl-0.2.0 → pydpm_xl-0.2.2}/py_dpm/dpm_xl/utils/serialization.py

@@ -542,10 +542,9 @@ class ASTToJSONVisitor(NodeVisitor):
 
         return result
 
-
 # Original serialization functions (kept for backward compatibility)
 import json
-from py_dpm.dpm_xl.ast import nodes
+from py_dpm.dpm_xl.ast import nodes as ASTObjects
 
 
 def expand_with_expression(node):
@@ -759,4 +758,4 @@ def ast_from_json_string(json_str):
         AST object instance
     """
     data = json.loads(json_str)
-    return deserialize_ast(data)
+    return deserialize_ast(data)

{pydpm_xl-0.2.0 → pydpm_xl-0.2.2/pydpm_xl.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydpm_xl
-Version: 0.2.0
+Version: 0.2.2
 Summary: Python library for DPM-XL data processing and analysis
 Author-email: "MeaningfulData S.L." <info@meaningfuldata.eu>
 License: GPL-3.0-or-later

{pydpm_xl-0.2.0 → pydpm_xl-0.2.2}/pydpm_xl.egg-info/SOURCES.txt

@@ -97,4 +97,5 @@ tests/test_get_tables_date_filter.py
 tests/test_get_tables_release_code.py
 tests/test_hierarchical_query.py
 tests/test_query_refactor.py
+tests/test_release_filters_semantic.py
 tests/test_semantic_release.py

{pydpm_xl-0.2.0 → pydpm_xl-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pydpm_xl"
-version = "0.2.0"
+version = "0.2.2"
 description = "Python library for DPM-XL data processing and analysis"
 authors = [
     {name = "MeaningfulData S.L.", email = "info@meaningfuldata.eu"}
@@ -52,7 +52,7 @@ exclude = []
 
 [tool.poetry]
 name = "pydpm_xl"
-version = "0.2.0"
+version = "0.2.2"
 description = "Python library for DPM-XL data processing and analysis"
 authors = ["MeaningfulData S.L. <info@meaningfuldata.eu>"]
 readme = "README.md"

{pydpm_xl-0.2.0 → pydpm_xl-0.2.2}/tests/test_cli_semantic.py

@@ -1,8 +1,9 @@
 import pytest
 from click.testing import CliRunner
-from py_dpm.cli.main import main, semantic
+from py_dpm.cli.main import main, semantic, syntax
 from unittest.mock import patch, MagicMock, MagicMock
 from py_dpm.api.dpm_xl.semantic import SemanticValidationResult
+from py_dpm.api.dpm_xl.syntax import SyntaxValidationResult
 
 
 @pytest.fixture
@@ -191,3 +192,59 @@ def test_semantic_conflict_flags(runner):
 
     assert result.exit_code != 0
     assert "Cannot provide both --release-id and --dpm-version" in result.output
+
+
+def test_syntax_valid_expression(runner):
+    """Test syntax command with a valid expression."""
+    expression = "{tC_01.00, r0100, c0010}"
+
+    with patch("py_dpm.cli.main.SyntaxAPI") as MockAPI:
+        mock_api_instance = MockAPI.return_value
+        mock_api_instance.validate_expression.return_value = SyntaxValidationResult(
+            is_valid=True,
+            error_message=None,
+            expression=expression,
+        )
+
+        result = runner.invoke(main, ["syntax", expression])
+
+        MockAPI.assert_called_once()
+        mock_api_instance.validate_expression.assert_called_once_with(expression)
+
+        assert result.exit_code == 0
+        assert "Syntax OK" in result.output
+
+
+def test_syntax_invalid_expression(runner):
+    """Test syntax command with an invalid expression."""
+    expression = "invalid_expression"
+
+    with patch("py_dpm.cli.main.SyntaxAPI") as MockAPI:
+        mock_api_instance = MockAPI.return_value
+        mock_api_instance.validate_expression.return_value = SyntaxValidationResult(
+            is_valid=False,
+            error_message="Syntax errors detected",
+            expression=expression,
+        )
+
+        result = runner.invoke(main, ["syntax", expression])
+
+        MockAPI.assert_called_once()
+        mock_api_instance.validate_expression.assert_called_once_with(expression)
+
+        assert result.exit_code == 0
+        assert "Syntax Error: Syntax errors detected" in result.output
+
+
+def test_syntax_unexpected_exception(runner):
+    """Test syntax command when an unexpected exception occurs."""
+    expression = "{tC_01.00}"
+
+    with patch("py_dpm.cli.main.SyntaxAPI") as MockAPI:
+        mock_api_instance = MockAPI.return_value
+        mock_api_instance.validate_expression.side_effect = Exception("Unexpected error")
+
+        result = runner.invoke(main, ["syntax", expression])
+
+        assert result.exit_code == 0
+        assert "Unexpected Error: Unexpected error" in result.output

pydpm_xl-0.2.2/tests/test_release_filters_semantic.py

@@ -0,0 +1,108 @@
+import pandas as pd
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from py_dpm.dpm.models import Base, TableVersion
+from py_dpm.dpm.queries.filters import filter_by_release
+from py_dpm.dpm_xl.ast import operands as operands_module
+
+
+def _make_session():
+    """Create a lightweight in-memory SQLAlchemy session for query compilation."""
+    engine = create_engine("sqlite:///:memory:")
+    Base.metadata.bind = engine
+    Session = sessionmaker(bind=engine)
+    return Session()
+
+
+def test_filter_by_release_uses_is_null_for_end_release():
+    """
+    Ensure filter_by_release uses SQLAlchemy .is_(None) semantics for end_col,
+    resulting in an 'IS NULL' predicate rather than '= NULL' in the SQL.
+
+    This is important for PostgreSQL, which is strict about boolean expressions
+    and NULL comparison semantics.
+    """
+    session = _make_session()
+
+    query = session.query(TableVersion)
+    filtered = filter_by_release(
+        query,
+        start_col=TableVersion.startreleaseid,
+        end_col=TableVersion.endreleaseid,
+        release_id=5,
+    )
+
+    sql = str(
+        filtered.statement.compile(
+            dialect=session.get_bind().dialect, compile_kwargs={"literal_binds": True}
+        )
+    ).upper()
+
+    # Ensure NULL handling uses IS NULL rather than = NULL
+    assert "IS NULL" in sql
+    assert "= NULL" not in sql
+
+
+def test_operands_check_headers_calls_filter_by_release_with_correct_args(monkeypatch):
+    """
+    Verify that OperandsChecking.check_headers wires filter_by_release correctly:
+    - start_col is TableVersion.startreleaseid
+    - end_col is TableVersion.endreleaseid
+    - release_id matches the instance's release_id
+    """
+    called = {}
+
+    def fake_filter_by_release(query, start_col, end_col, release_id=None, release_code=None):
+        called["query"] = query
+        called["start_col"] = start_col
+        called["end_col"] = end_col
+        called["release_id"] = release_id
+        called["release_code"] = release_code
+        return query
+
+    monkeypatch.setattr(operands_module, "filter_by_release", fake_filter_by_release)
+
+    # Stub out the pandas helpers used inside check_headers so that no real DB
+    # access is attempted.
+    import py_dpm.dpm.models as models
+
+    monkeypatch.setattr(
+        models,
+        "_compile_query_for_pandas",
+        lambda stmt, session: stmt,
+    )
+
+    def fake_read_sql(sql, session):
+        # Return an empty DataFrame with the expected columns so that
+        # check_headers completes without touching self.operands.
+        return pd.DataFrame(
+            columns=[
+                "Code",
+                "StartReleaseID",
+                "EndReleaseID",
+                "Direction",
+                "HasOpenRows",
+                "HasOpenColumns",
+                "HasOpenSheets",
+            ]
+        )
+
+    monkeypatch.setattr(models, "_read_sql_with_connection", fake_read_sql)
+
+    session = _make_session()
+
+    # Create a minimal OperandsChecking instance without running its __init__,
+    # since that would require a fully-populated AST and database.
+    oc = object.__new__(operands_module.OperandsChecking)
+    oc.session = session
+    oc.release_id = 7
+    oc.tables = {"DummyTable": {}}  # Only keys are needed by check_headers
+
+    operands_module.OperandsChecking.check_headers(oc)
+
+    assert called["start_col"] is TableVersion.startreleaseid
+    assert called["end_col"] is TableVersion.endreleaseid
+    assert called["release_id"] == 7
+    assert called["release_code"] is None
+