pydpm_xl 0.2.8__tar.gz → 0.2.10__tar.gz

{pydpm_xl-0.2.8/pydpm_xl.egg-info → pydpm_xl-0.2.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydpm_xl
-Version: 0.2.8
+Version: 0.2.10
 Summary: Python library for DPM-XL data processing and analysis
 Author-email: "MeaningfulData S.L." <info@meaningfuldata.eu>
 License: GPL-3.0-or-later
@@ -210,22 +210,21 @@ else:
     print(f"Error: {result.error}")
 ```
 
-#### Complete AST Generation
+#### Validations Script Generation
 
 ```python
-from py_dpm.api import generate_complete_ast, generate_enriched_ast
+from py_dpm.api import generate_validations_script
 
-# Generate complete AST with data fields
-result = generate_complete_ast("{tT_01.00, r0010, c0010}", release_id=123)
-if result.success:
-    print(f"AST: {result.ast}")
-    print(f"Data fields: {result.data_fields}")
+# Generate engine-ready validations script
+result = generate_validations_script(
+    "{tT_01.00, r0010, c0010}",
+    database_path="data.db",
+    release_code="4.2"
+)
+if result["success"]:
+    print(f"Enriched AST: {result['enriched_ast']}")
 else:
-    print(f"Errors: {result.errors}")
-
-# Generate enriched AST (ready for execution engine)
-enriched = generate_enriched_ast("{tT_01.00, r0010, c0010}", release_id=123)
-print(f"Enriched AST: {enriched}")
+    print(f"Error: {result['error']}")
 ```
 
 #### Migration

{pydpm_xl-0.2.8 → pydpm_xl-0.2.10}/README.md

@@ -177,22 +177,21 @@ else:
     print(f"Error: {result.error}")
 ```
 
-#### Complete AST Generation
+#### Validations Script Generation
 
 ```python
-from py_dpm.api import generate_complete_ast, generate_enriched_ast
+from py_dpm.api import generate_validations_script
 
-# Generate complete AST with data fields
-result = generate_complete_ast("{tT_01.00, r0010, c0010}", release_id=123)
-if result.success:
-    print(f"AST: {result.ast}")
-    print(f"Data fields: {result.data_fields}")
+# Generate engine-ready validations script
+result = generate_validations_script(
+    "{tT_01.00, r0010, c0010}",
+    database_path="data.db",
+    release_code="4.2"
+)
+if result["success"]:
+    print(f"Enriched AST: {result['enriched_ast']}")
 else:
-    print(f"Errors: {result.errors}")
-
-# Generate enriched AST (ready for execution engine)
-enriched = generate_enriched_ast("{tT_01.00, r0010, c0010}", release_id=123)
-print(f"Enriched AST: {enriched}")
+    print(f"Error: {result['error']}")
 ```
 
 #### Migration

{pydpm_xl-0.2.8 → pydpm_xl-0.2.10}/py_dpm/__init__.py

@@ -41,7 +41,7 @@ Available packages:
 - pydpm.api: Main APIs for migration, syntax, and semantic analysis
 """
 
-__version__ = "0.2.8"
+__version__ = "0.2.10"
 __author__ = "MeaningfulData S.L."
 __email__ = "info@meaningfuldata.eu"
 __license__ = "GPL-3.0-or-later"

{pydpm_xl-0.2.8 → pydpm_xl-0.2.10}/py_dpm/api/__init__.py

@@ -11,10 +11,7 @@ from py_dpm.api.dpm_xl import (
     SemanticAPI,
     ASTGeneratorAPI,
     OperationScopesAPI,
-    generate_complete_ast,
-    generate_enriched_ast,
-    enrich_ast_with_metadata,
-    parse_with_data_fields,
+    generate_validations_script,
 )
 
 # Import from general DPM API
@@ -40,9 +37,6 @@ __all__ = [
     "SemanticAPI",
     "ASTGeneratorAPI",
     "OperationScopesAPI",
-    # Complete AST functions (backwards compatibility)
-    "generate_complete_ast",
-    "generate_enriched_ast",
-    "enrich_ast_with_metadata",
-    "parse_with_data_fields",
+    # Standalone function
+    "generate_validations_script",
 ]

{pydpm_xl-0.2.8 → pydpm_xl-0.2.10}/py_dpm/api/dpm_xl/__init__.py

@@ -9,13 +9,7 @@ from py_dpm.api.dpm_xl.semantic import SemanticAPI
 from py_dpm.api.dpm_xl.ast_generator import ASTGeneratorAPI
 from py_dpm.api.dpm_xl.operation_scopes import OperationScopesAPI
 
-# Backwards-compatible standalone functions (delegate to ASTGeneratorAPI)
-from py_dpm.api.dpm_xl.complete_ast import (
-    generate_complete_ast,
-    generate_enriched_ast,
-    enrich_ast_with_metadata,
-    parse_with_data_fields,
-)
+from py_dpm.api.dpm_xl.complete_ast import generate_validations_script
 
 __all__ = [
     # Class-based APIs
@@ -23,9 +17,6 @@ __all__ = [
     "SemanticAPI",
     "ASTGeneratorAPI",
     "OperationScopesAPI",
-    # Standalone functions (backwards compatibility)
-    "generate_complete_ast",
-    "generate_enriched_ast",
-    "enrich_ast_with_metadata",
-    "parse_with_data_fields",
+    # Standalone function
+    "generate_validations_script",
 ]

{pydpm_xl-0.2.8 → pydpm_xl-0.2.10}/py_dpm/api/dpm_xl/ast_generator.py

@@ -32,7 +32,7 @@ class ASTGeneratorAPI:
        - Returns: AST with data fields populated (datapoint IDs, operand references)
        - Use for: AST analysis with complete metadata, matching json_scripts/*.json format
 
-    3. **Enriched AST** (generate_enriched_ast):
+    3. **Validations Script** (generate_validations_script):
        - Requires database connection
        - Extends complete AST with framework structure for execution engines
        - Returns: Engine-ready AST with operations, variables, tables, preconditions sections
@@ -90,7 +90,7 @@ class ASTGeneratorAPI:
 
         **What you DON'T get:**
         - Data fields (datapoint IDs, operand references) - use generate_complete_ast()
-        - Framework structure - use generate_enriched_ast()
+        - Framework structure - use generate_validations_script()
 
         Args:
             expression: DPM-XL expression string
@@ -347,7 +347,7 @@ class ASTGeneratorAPI:
             return [(expressions, "default_code", None)]
         return expressions
 
-    def generate_enriched_ast(
+    def generate_validations_script(
         self,
         expressions: Union[str, List[Tuple[str, str, Optional[str]]]],
         release_code: Optional[str] = None,
@@ -360,9 +360,9 @@ class ASTGeneratorAPI:
         module_version_number: Optional[str] = None,
     ) -> Dict[str, Any]:
         """
-        Generate enriched, engine-ready AST with framework structure (Level 3).
+        Generate validations script with engine-ready AST and framework structure.
 
-        This extends generate_complete_ast() by wrapping the complete AST in an engine-ready
+        This method generates the complete validations script by wrapping ASTs in an engine-ready
         framework structure with operations, variables, tables, and preconditions sections.
         This is the format required by business rule execution engines.
 
@@ -370,7 +370,7 @@ class ASTGeneratorAPI:
         expression/operation/precondition tuples for generating scripts with multiple operations.
 
         **What you get:**
-        - Everything from generate_complete_ast() PLUS:
+        - Complete AST with data fields PLUS:
         - Framework structure: operations, variables, tables, preconditions
         - Module metadata: version, release info, dates
         - Dependency information (including cross-module dependencies)
@@ -428,14 +428,14 @@ class ASTGeneratorAPI:
 
         Example:
             >>> generator = ASTGeneratorAPI(database_path="data.db")
-            >>> # Single expression (backward compatible)
-            >>> result = generator.generate_enriched_ast(
+            >>> # Single expression
+            >>> result = generator.generate_validations_script(
             ...     "{tF_01.00, r0010, c0010}",
             ...     release_code="4.2",
             ... )
             >>>
             >>> # Multiple expressions with operations and preconditions
-            >>> result = generator.generate_enriched_ast(
+            >>> result = generator.generate_validations_script(
             ...     [
             ...         ("{tF_01.00, r0010, c0010} = 0", "v1234_m", None),
             ...         ("{tF_01.00, r0020, c0010} > 0", "v1235_m", "{v_F_44_04}"),
@@ -984,6 +984,7 @@ class ASTGeneratorAPI:
         """
         from py_dpm.dpm.utils import get_engine
         from py_dpm.api.dpm import DataDictionaryAPI
+        from py_dpm.api.dpm_xl.operation_scopes import OperationScopesAPI
 
         # Initialize database connection
         engine = get_engine(database_path=self.database_path, connection_url=self.connection_url)
@@ -1017,6 +1018,12 @@ class ASTGeneratorAPI:
             connection_url=self.connection_url
         )
 
+        # Initialize OperationScopesAPI once for all expressions (performance optimization)
+        scopes_api = OperationScopesAPI(
+            database_path=self.database_path,
+            connection_url=self.connection_url
+        )
+
         # Primary module info will be determined from the first expression or module_code
         primary_module_info = None
         namespace = None
@@ -1035,6 +1042,21 @@ class ASTGeneratorAPI:
                 complete_ast = complete_result["ast"]
                 context = complete_result.get("context") or table_context
 
+                # Get tables with modules for this expression FIRST (reuse scopes_api from outer scope)
+                # This is done before _get_primary_module_info to pass precomputed values
+                tables_with_modules = scopes_api.get_tables_with_metadata_from_expression(
+                    expression=expression,
+                    release_id=release_id
+                )
+
+                # Calculate scope_result once (avoid duplicate calls in other methods)
+                scope_result = scopes_api.calculate_scopes_from_expression(
+                    expression=expression,
+                    release_id=release_id,
+                    read_only=True
+                )
+                all_tables_with_modules.extend(tables_with_modules)
+
                 # Get primary module info from first expression (or use module_code)
                 if primary_module_info is None:
                     primary_module_info = self._get_primary_module_info(
@@ -1042,6 +1064,9 @@ class ASTGeneratorAPI:
                         primary_module_vid=primary_module_vid,
                         release_id=release_id,
                         module_code=module_code,
+                        # Performance optimization: pass precomputed values
+                        tables_with_modules=tables_with_modules,
+                        scopes_api=scopes_api,
                     )
                     namespace = primary_module_info.get("module_uri", "default_module")
 
@@ -1066,18 +1091,6 @@ class ASTGeneratorAPI:
                 # Clean extra fields from data entries
                 self._clean_ast_data_entries(ast_with_coords)
 
-                # Get tables with modules for this expression
-                from py_dpm.api.dpm_xl.operation_scopes import OperationScopesAPI
-                scopes_api = OperationScopesAPI(
-                    database_path=self.database_path,
-                    connection_url=self.connection_url
-                )
-                tables_with_modules = scopes_api.get_tables_with_metadata_from_expression(
-                    expression=expression,
-                    release_id=release_id
-                )
-                all_tables_with_modules.extend(tables_with_modules)
-
                 # Build mapping of table_code -> module_vid
                 # Prefer the module VID that matches the detected primary module
                 table_to_module = {}
@@ -1179,6 +1192,10 @@ class ASTGeneratorAPI:
                     operation_code=operation_code,
                     release_id=release_id,
                     preferred_module_dependencies=preferred_module_dependencies,
+                    # Performance optimization: pass precomputed values to avoid redundant work
+                    tables_with_modules=tables_with_modules,
+                    scopes_api=scopes_api,
+                    scope_result=scope_result,
                 )
 
                 # Merge dependency modules (avoid table duplicates)
@@ -1313,6 +1330,8 @@ class ASTGeneratorAPI:
         primary_module_vid: Optional[int],
         release_id: Optional[int],
         module_code: Optional[str] = None,
+        tables_with_modules: Optional[List[Dict[str, Any]]] = None,
+        scopes_api: Optional[Any] = None,
     ) -> Dict[str, Any]:
         """
         Detect and return metadata for the primary module from the expression.
@@ -1323,6 +1342,10 @@ class ASTGeneratorAPI:
             release_id: Optional release ID for filtering
             module_code: Optional module code (e.g., "FINREP9") - takes precedence over
                 primary_module_vid if provided
+            tables_with_modules: Optional precomputed tables with module metadata
+                (performance optimization to avoid redundant database queries)
+            scopes_api: Optional precomputed OperationScopesAPI instance
+                (performance optimization to reuse database connections)
 
         Returns:
             Dict with module_uri, module_code, module_version, framework_code,
@@ -1341,20 +1364,28 @@ class ASTGeneratorAPI:
             "module_vid": None,
         }
 
+        # Track if we created the scopes_api locally (need to close it)
+        local_scopes_api = False
+
         try:
-            scopes_api = OperationScopesAPI(
-                database_path=self.database_path,
-                connection_url=self.connection_url
-            )
+            # Reuse provided scopes_api or create a new one
+            if scopes_api is None:
+                scopes_api = OperationScopesAPI(
+                    database_path=self.database_path,
+                    connection_url=self.connection_url
+                )
+                local_scopes_api = True
 
-            # Get tables with module metadata from expression
-            tables_with_modules = scopes_api.get_tables_with_metadata_from_expression(
-                expression=expression,
-                release_id=release_id
-            )
+            # Reuse provided tables_with_modules or fetch if not available
+            if tables_with_modules is None:
+                tables_with_modules = scopes_api.get_tables_with_metadata_from_expression(
+                    expression=expression,
+                    release_id=release_id
+                )
 
             if not tables_with_modules:
-                scopes_api.close()
+                if local_scopes_api:
+                    scopes_api.close()
                 return default_info
 
             # Determine primary module
@@ -1408,7 +1439,8 @@ class ASTGeneratorAPI:
                         to_date = module.get("to_reference_date", to_date)
                         break
 
-            scopes_api.close()
+            if local_scopes_api:
+                scopes_api.close()
 
             return {
                 "module_uri": module_uri or "default_module",
@@ -1864,6 +1896,9 @@ class ASTGeneratorAPI:
         operation_code: str,
         release_id: Optional[int] = None,
         preferred_module_dependencies: Optional[List[str]] = None,
+        tables_with_modules: Optional[List[Dict[str, Any]]] = None,
+        scopes_api: Optional[Any] = None,
+        scope_result: Optional[Any] = None,
     ) -> tuple:
         """
         Detect cross-module dependencies for a single expression.
@@ -1879,6 +1914,12 @@ class ASTGeneratorAPI:
             release_id: Optional release ID for filtering
             preferred_module_dependencies: Optional list of module codes to prefer when
                 a table belongs to multiple modules
+            tables_with_modules: Optional precomputed tables with module metadata
+                (performance optimization to avoid redundant database queries)
+            scopes_api: Optional precomputed OperationScopesAPI instance
+                (performance optimization to reuse database connections)
+            scope_result: Optional precomputed scope result from calculate_scopes_from_expression
+                (performance optimization to avoid redundant computation)
 
         Returns:
             Tuple of (dependency_modules, cross_instance_dependencies)
@@ -1889,24 +1930,28 @@ class ASTGeneratorAPI:
         from py_dpm.dpm.queries.explorer_queries import ExplorerQuery
         import logging
 
-        scopes_api = OperationScopesAPI(
-            database_path=self.database_path,
-            connection_url=self.connection_url
-        )
+        # Reuse provided scopes_api or create a new one
+        if scopes_api is None:
+            scopes_api = OperationScopesAPI(
+                database_path=self.database_path,
+                connection_url=self.connection_url
+            )
 
         try:
-            # Get tables with module info (includes module_version)
-            tables_with_modules = scopes_api.get_tables_with_metadata_from_expression(
-                expression=expression,
-                release_id=release_id
-            )
+            # Reuse provided tables_with_modules or fetch if not available
+            if tables_with_modules is None:
+                tables_with_modules = scopes_api.get_tables_with_metadata_from_expression(
+                    expression=expression,
+                    release_id=release_id
+                )
 
-            # Check if cross-module
-            scope_result = scopes_api.calculate_scopes_from_expression(
-                expression=expression,
-                release_id=release_id,
-                read_only=True
-            )
+            # Reuse provided scope_result or compute if not available
+            if scope_result is None:
+                scope_result = scopes_api.calculate_scopes_from_expression(
+                    expression=expression,
+                    release_id=release_id,
+                    read_only=True
+                )
 
             if scope_result.has_error or not scope_result.is_cross_module:
                 return {}, []

pydpm_xl-0.2.10/py_dpm/api/dpm_xl/complete_ast.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""
+Validations Script API - Generate engine-ready ASTs for validation frameworks.
+
+This module provides a standalone function that delegates to ASTGeneratorAPI.
+
+For direct class usage:
+    from py_dpm.api.dpm_xl import ASTGeneratorAPI
+
+    generator = ASTGeneratorAPI(database_path="data.db")
+    result = generator.generate_validations_script(expressions)
+"""
+
+from typing import Dict, Any, Optional, List, Union, Tuple
+from py_dpm.api.dpm_xl.ast_generator import ASTGeneratorAPI
+
+
+def generate_validations_script(
+    expressions: Union[str, List[Tuple[str, str, Optional[str]]]],
+    database_path: Optional[str] = None,
+    connection_url: Optional[str] = None,
+    release_code: Optional[str] = None,
+    table_context: Optional[Dict[str, Any]] = None,
+    release_id: Optional[int] = None,
+    primary_module_vid: Optional[int] = None,
+    module_code: Optional[str] = None,
+    preferred_module_dependencies: Optional[List[str]] = None,
+) -> Dict[str, Any]:
+    """
+    Generate validations script with engine-ready AST from DPM-XL expression(s).
+
+    This function delegates to ASTGeneratorAPI.generate_validations_script().
+
+    Supports both single expressions and multiple expression/operation/precondition
+    tuples for generating scripts with multiple operations.
+
+    Args:
+        expressions: Either a single DPM-XL expression string,
+            or a list of tuples: [(expression, operation_code, precondition), ...].
+            Each tuple contains:
+            - expression (str): The DPM-XL expression (required)
+            - operation_code (str): The operation code (required)
+            - precondition (Optional[str]): Optional precondition reference (e.g., {v_F_44_04})
+        database_path: Path to SQLite database (or None for PostgreSQL)
+        connection_url: PostgreSQL connection URL (takes precedence over database_path)
+        release_code: DPM release code (e.g., "4.0", "4.1", "4.2")
+        table_context: Optional table context dict with keys: 'table', 'columns', 'rows', 'sheets', 'default', 'interval'
+        release_id: Optional release ID to filter database lookups by specific release.
+            If None, uses all available data (release-agnostic).
+        primary_module_vid: Optional module version ID of the module being exported.
+            When provided, enables detection of cross-module dependencies.
+        module_code: Optional module code (e.g., "FINREP9") to specify the main module.
+        preferred_module_dependencies: Optional list of module codes to prefer when
+            multiple dependency scopes are possible.
+
+    Returns:
+        dict: {
+            'success': bool,
+            'enriched_ast': dict,  # Engine-ready AST with framework structure
+            'error': str           # Error message if failed
+        }
+
+    Example:
+        >>> # Single expression
+        >>> result = generate_validations_script(
+        ...     "{tF_01.00, r0010, c0010}",
+        ...     database_path="data.db",
+        ...     release_code="4.2",
+        ... )
+        >>>
+        >>> # Multiple expressions
+        >>> result = generate_validations_script(
+        ...     [
+        ...         ("{tF_01.00, r0010, c0010} = 0", "v1234_m", None),
+        ...         ("{tF_01.00, r0020, c0010} > 0", "v1235_m", "{v_F_44_04}"),
+        ...     ],
+        ...     database_path="data.db",
+        ...     release_code="4.2",
+        ...     module_code="FINREP9",
+        ... )
+    """
+    generator = ASTGeneratorAPI(
+        database_path=database_path,
+        connection_url=connection_url,
+        enable_semantic_validation=True
+    )
+    return generator.generate_validations_script(
+        expressions=expressions,
+        release_code=release_code,
+        table_context=table_context,
+        release_id=release_id,
+        primary_module_vid=primary_module_vid,
+        module_code=module_code,
+        preferred_module_dependencies=preferred_module_dependencies,
+    )

{pydpm_xl-0.2.8 → pydpm_xl-0.2.10}/py_dpm/dpm/migration.py

@@ -103,14 +103,16 @@ def _extract_with_pyodbc(access_file):
         import pyodbc
     except ImportError:
         raise Exception("pyodbc not available")
-    
+
+    import decimal
+
     # Try different Access drivers
     drivers_to_try = [
         r'DRIVER={Microsoft Access Driver (*.mdb, *.accdb)};',
         r'DRIVER={Microsoft Access Driver (*.mdb)};',
         r'DRIVER={MDBTools};'
     ]
-    
+
     conn = None
     for driver in drivers_to_try:
         try:
@@ -120,10 +122,10 @@ def _extract_with_pyodbc(access_file):
             break
         except pyodbc.Error:
             continue
-    
+
     if not conn:
         raise Exception("No suitable ODBC driver found for Access database")
-    
+
     try:
         # Get all table names
         cursor = conn.cursor()
@@ -132,63 +134,56 @@ def _extract_with_pyodbc(access_file):
             table_name = table_info.table_name
             if not table_name.startswith('MSys'):  # Skip system tables
                 tables.append(table_name)
-        
+
         data = {}
-        STRING_COLUMNS = ["row", "column", "sheet"]
-        
+
         # Extract each table
         for table_name in tables:
             print(table_name)
             try:
                 cursor.execute(f"SELECT * FROM [{table_name}]")
-                columns = [column[0] for column in cursor.description]
+
+                # Get column metadata from cursor.description
+                # Each entry is: (name, type_code, display_size, internal_size, precision, scale, null_ok)
+                # type_code is a Python type (str, int, float, decimal.Decimal, etc.)
+                column_info = []
+                for col_desc in cursor.description:
+                    col_name = col_desc[0]
+                    col_type = col_desc[1]  # Python type from ODBC metadata
+                    column_info.append((col_name, col_type))
+
+                columns = [info[0] for info in column_info]
                 rows = cursor.fetchall()
-                
+
                 if rows:
                     # Convert to DataFrame
                     df = pd.DataFrame([list(row) for row in rows], columns=columns)
 
-                    # Apply same dtype conversion logic as mdb-tools method
-                    # Start with all strings, but preserve None as actual None (not string 'None')
-                    for col in df.columns:
-                        df[col] = df[col].astype(object)
-                        mask = df[col].notna()
-                        df.loc[mask, col] = df.loc[mask, col].astype(str)
-
-                    numeric_columns = []
-                    for column in df.columns:
-                        if column in STRING_COLUMNS:
-                            continue
-                        try:
-                            # Convert to numeric and check if any values start with '0' (except '0' itself)
-                            # Only check string values for leading zeros
-                            string_mask = df[column].astype(str).str.match(r'^0\d+', na=False)
-                            has_leading_zeros = string_mask.any()
-
-                            # Test numeric conversion
-                            numeric_series = pd.to_numeric(df[column], errors='coerce')
-
-                            if not has_leading_zeros and not numeric_series.isna().all():
-                                numeric_columns.append(column)
-                        except Exception:
-                            continue
-
-                    # Convert only the identified numeric columns
-                    for col in numeric_columns:
-                        try:
-                            df[col] = pd.to_numeric(df[col], errors='coerce')
-                        except (ValueError, TypeError):
-                            # Keep as string if conversion fails
-                            pass
-                    
+                    # Use the actual column types from Access schema metadata
+                    # instead of inferring from data values (fixes Windows vs Linux inconsistency)
+                    numeric_types = (int, float, decimal.Decimal)
+
+                    for col_name, col_type in column_info:
+                        if col_type in numeric_types:
+                            # Column is defined as numeric in Access schema - convert to numeric
+                            try:
+                                df[col_name] = pd.to_numeric(df[col_name], errors='coerce')
+                            except (ValueError, TypeError):
+                                pass
+                        else:
+                            # Column is defined as text/other in Access schema - keep as string
+                            df[col_name] = df[col_name].astype(object)
+                            mask = df[col_name].notna()
+                            df.loc[mask, col_name] = df.loc[mask, col_name].astype(str)
+
                     data[table_name] = df
-                
+
             except Exception as e:
                 print(f"Error processing table {table_name}: {e}", file=sys.stderr)
                 continue
-        
+
         return data
-        
+
     finally:
         conn.close()