pydpm_xl 0.1.10__py3-none-any.whl

py_dpm/api/data_dictionary_validation.py

@@ -0,0 +1,577 @@
+"""
+Data Dictionary Validation API
+
+This module provides methods to identify and validate data dictionary issues
+that can cause semantic validation failures during DPM-XL transpilation.
+"""
+
+from typing import Dict, List, Set, Optional, Any, Tuple
+from dataclasses import dataclass
+from enum import Enum
+import re
+from sqlalchemy import text
+
+from py_dpm.db_utils import get_session, get_engine
+from py_dpm.models import *
+
+
+class ValidationIssueType(Enum):
+    """Types of data dictionary validation issues."""
+    MISSING_TABLE = "missing_table"
+    MISSING_COLUMN = "missing_column"
+    MISSING_ROW = "missing_row"
+    MISSING_SHEET = "missing_sheet"
+    MISSING_VARIABLE = "missing_variable"
+    INVALID_REFERENCE = "invalid_reference"
+    TYPE_MISMATCH = "type_mismatch"
+    CONFIGURATION_ERROR = "configuration_error"
+
+
+@dataclass
+class ValidationIssue:
+    """
+    Represents a data dictionary validation issue.
+    
+    Attributes:
+        issue_type (ValidationIssueType): Type of the issue
+        description (str): Human-readable description of the issue
+        affected_element (str): The specific element that has the issue
+        suggested_fix (Optional[str]): Suggested fix for the issue
+        severity (str): Severity level ('error', 'warning', 'info')
+        code (Optional[str]): Error code for programmatic handling
+    """
+    issue_type: ValidationIssueType
+    description: str
+    affected_element: str
+    suggested_fix: Optional[str] = None
+    severity: str = "error"
+    code: Optional[str] = None
+
+
+@dataclass
+class CellReference:
+    """Represents a parsed cell reference from DPM-XL expression."""
+    table: str
+    rows: List[str]
+    columns: List[str]
+    sheets: List[str]
+
+
+class DataDictionaryValidator:
+    """
+    Main class for validating data dictionary consistency and completeness.
+    
+    This class provides methods to detect issues that would cause semantic
+    validation failures during DPM-XL transpilation.
+    """
+    
+    def __init__(self):
+        """Initialize the Data Dictionary Validator."""
+        get_engine()
+        self.session = get_session()
+        self._table_cache = {}
+        self._column_cache = {}
+        self._row_cache = {}
+        self._sheet_cache = {}
+        
+    def validate_expression_references(self, dpm_xl_expression: str) -> List[ValidationIssue]:
+        """
+        Validate all cell references in a DPM-XL expression.
+        
+        Args:
+            dpm_xl_expression (str): The DPM-XL expression to validate
+            
+        Returns:
+            List[ValidationIssue]: List of validation issues found
+        """
+        issues = []
+        
+        try:
+            # Parse cell references from the expression
+            cell_refs = self._parse_cell_references(dpm_xl_expression)
+            
+            for cell_ref in cell_refs:
+                # Validate table exists
+                table_issues = self.validate_table_exists(cell_ref.table)
+                issues.extend(table_issues)
+                
+                # If table exists, validate other components
+                if not table_issues:
+                    # Validate columns
+                    column_issues = self.validate_columns_exist(cell_ref.table, cell_ref.columns)
+                    issues.extend(column_issues)
+                    
+                    # Validate rows
+                    row_issues = self.validate_rows_exist(cell_ref.table, cell_ref.rows)
+                    issues.extend(row_issues)
+                    
+                    # Validate sheets
+                    sheet_issues = self.validate_sheets_exist(cell_ref.table, cell_ref.sheets)
+                    issues.extend(sheet_issues)
+                    
+        except Exception as e:
+            issues.append(ValidationIssue(
+                issue_type=ValidationIssueType.CONFIGURATION_ERROR,
+                description=f"Error parsing expression: {str(e)}",
+                affected_element=dpm_xl_expression[:50] + "..." if len(dpm_xl_expression) > 50 else dpm_xl_expression,
+                severity="error",
+                code="PARSE_ERROR"
+            ))
+            
+        return issues
+    
+    def validate_table_exists(self, table_name: str) -> List[ValidationIssue]:
+        """
+        Validate that a table exists in the data dictionary.
+        
+        Args:
+            table_name (str): Name of the table to validate
+            
+        Returns:
+            List[ValidationIssue]: List of validation issues found
+        """
+        issues = []
+        
+        try:
+            # Check cache first
+            if table_name in self._table_cache:
+                return self._table_cache[table_name]
+            
+            # Query the database for the table using the correct schema
+            # The actual schema uses table_code instead of table_name
+            tables = self.session.execute(
+                text("SELECT DISTINCT table_code FROM datapoints WHERE table_code = :table_code"),
+                {"table_code": table_name}
+            ).fetchall()
+            
+            if not tables:
+                issue = ValidationIssue(
+                    issue_type=ValidationIssueType.MISSING_TABLE,
+                    description=f"Table '{table_name}' was not found in the data dictionary",
+                    affected_element=table_name,
+                    suggested_fix=f"Add table '{table_name}' to the data dictionary or check the table name spelling",
+                    severity="error",
+                    code="TABLE_NOT_FOUND"
+                )
+                issues.append(issue)
+            
+            # Cache the result
+            self._table_cache[table_name] = issues
+            
+        except Exception as e:
+            issues.append(ValidationIssue(
+                issue_type=ValidationIssueType.CONFIGURATION_ERROR,
+                description=f"Error checking table '{table_name}': {str(e)}",
+                affected_element=table_name,
+                severity="error",
+                code="TABLE_CHECK_ERROR"
+            ))
+        
+        return issues
+    
+    def validate_columns_exist(self, table_name: str, columns: List[str]) -> List[ValidationIssue]:
+        """
+        Validate that columns exist for a table.
+        
+        Args:
+            table_name (str): Name of the table
+            columns (List[str]): List of column names/patterns to validate
+            
+        Returns:
+            List[ValidationIssue]: List of validation issues found
+        """
+        issues = []
+        
+        try:
+            cache_key = f"{table_name}:{':'.join(columns)}"
+            if cache_key in self._column_cache:
+                return self._column_cache[cache_key]
+            
+            for column in columns:
+                # Skip wildcards and ranges for now - these need special handling
+                if column in ['*'] or '-' in column:
+                    continue
+                    
+                # Check if specific column exists
+                column_exists = self.session.execute(
+                    text("SELECT COUNT(*) FROM datapoints WHERE table_code = :table_code AND column_code = :column_code"),
+                    {"table_code": table_name, "column_code": column}
+                ).fetchone()
+                
+                if not column_exists or column_exists[0] == 0:
+                    issue = ValidationIssue(
+                        issue_type=ValidationIssueType.MISSING_COLUMN,
+                        description=f"Column '{column}' not found in table '{table_name}'",
+                        affected_element=f"{table_name}.{column}",
+                        suggested_fix=f"Add column '{column}' to table '{table_name}' or check the column name",
+                        severity="error",
+                        code="COLUMN_NOT_FOUND"
+                    )
+                    issues.append(issue)
+            
+            self._column_cache[cache_key] = issues
+            
+        except Exception as e:
+            issues.append(ValidationIssue(
+                issue_type=ValidationIssueType.CONFIGURATION_ERROR,
+                description=f"Error checking columns for table '{table_name}': {str(e)}",
+                affected_element=f"{table_name}.[{','.join(columns)}]",
+                severity="error",
+                code="COLUMN_CHECK_ERROR"
+            ))
+        
+        return issues
+    
+    def validate_rows_exist(self, table_name: str, rows: List[str]) -> List[ValidationIssue]:
+        """
+        Validate that rows exist for a table.
+        
+        Args:
+            table_name (str): Name of the table
+            rows (List[str]): List of row names/patterns to validate
+            
+        Returns:
+            List[ValidationIssue]: List of validation issues found
+        """
+        issues = []
+        
+        try:
+            cache_key = f"{table_name}:rows:{':'.join(rows)}"
+            if cache_key in self._row_cache:
+                return self._row_cache[cache_key]
+            
+            for row in rows:
+                # Skip wildcards and ranges for now
+                if row in ['*'] or '-' in row:
+                    continue
+                    
+                # Check if specific row exists
+                row_exists = self.session.execute(
+                    text("SELECT COUNT(*) FROM datapoints WHERE table_code = :table_code AND row_code = :row_code"),
+                    {"table_code": table_name, "row_code": row}
+                ).fetchone()
+                
+                if not row_exists or row_exists[0] == 0:
+                    issue = ValidationIssue(
+                        issue_type=ValidationIssueType.MISSING_ROW,
+                        description=f"Row '{row}' not found in table '{table_name}'",
+                        affected_element=f"{table_name}.{row}",
+                        suggested_fix=f"Add row '{row}' to table '{table_name}' or check the row name",
+                        severity="warning",  # Rows might be more flexible
+                        code="ROW_NOT_FOUND"
+                    )
+                    issues.append(issue)
+            
+            self._row_cache[cache_key] = issues
+            
+        except Exception as e:
+            issues.append(ValidationIssue(
+                issue_type=ValidationIssueType.CONFIGURATION_ERROR,
+                description=f"Error checking rows for table '{table_name}': {str(e)}",
+                affected_element=f"{table_name}.[{','.join(rows)}]",
+                severity="error",
+                code="ROW_CHECK_ERROR"
+            ))
+        
+        return issues
+    
+    def validate_sheets_exist(self, table_name: str, sheets: List[str]) -> List[ValidationIssue]:
+        """
+        Validate that sheets exist for a table.
+        
+        Args:
+            table_name (str): Name of the table
+            sheets (List[str]): List of sheet names/patterns to validate
+            
+        Returns:
+            List[ValidationIssue]: List of validation issues found
+        """
+        issues = []
+        
+        try:
+            cache_key = f"{table_name}:sheets:{':'.join(sheets)}"
+            if cache_key in self._sheet_cache:
+                return self._sheet_cache[cache_key]
+            
+            for sheet in sheets:
+                # Skip wildcards for now
+                if sheet in ['*']:
+                    # Check if any sheets exist for this table
+                    sheet_count = self.session.execute(
+                        text("SELECT COUNT(DISTINCT sheet_code) FROM datapoints WHERE table_code = :table_code AND sheet_code IS NOT NULL AND sheet_code != ''"),
+                        {"table_code": table_name}
+                    ).fetchone()
+                    
+                    if not sheet_count or sheet_count[0] == 0:
+                        issue = ValidationIssue(
+                            issue_type=ValidationIssueType.MISSING_SHEET,
+                            description=f"No sheets found for table '{table_name}' but s* wildcard used",
+                            affected_element=f"{table_name}.s*",
+                            suggested_fix=f"Add sheet definitions for table '{table_name}' or remove s* wildcard",
+                            severity="error",
+                            code="NO_SHEETS_FOR_WILDCARD"
+                        )
+                        issues.append(issue)
+                    continue
+                
+                # Check if specific sheet exists
+                sheet_exists = self.session.execute(
+                    text("SELECT COUNT(*) FROM datapoints WHERE table_code = :table_code AND sheet_code = :sheet_code"),
+                    {"table_code": table_name, "sheet_code": sheet}
+                ).fetchone()
+                
+                if not sheet_exists or sheet_exists[0] == 0:
+                    issue = ValidationIssue(
+                        issue_type=ValidationIssueType.MISSING_SHEET,
+                        description=f"Sheet '{sheet}' not found in table '{table_name}'",
+                        affected_element=f"{table_name}.{sheet}",
+                        suggested_fix=f"Add sheet '{sheet}' to table '{table_name}' or check the sheet name",
+                        severity="error",
+                        code="SHEET_NOT_FOUND"
+                    )
+                    issues.append(issue)
+            
+            self._sheet_cache[cache_key] = issues
+            
+        except Exception as e:
+            issues.append(ValidationIssue(
+                issue_type=ValidationIssueType.CONFIGURATION_ERROR,
+                description=f"Error checking sheets for table '{table_name}': {str(e)}",
+                affected_element=f"{table_name}.[{','.join(sheets)}]",
+                severity="error",
+                code="SHEET_CHECK_ERROR"
+            ))
+        
+        return issues
+    
+    def validate_variables_exist(self, variable_names: List[str]) -> List[ValidationIssue]:
+        """
+        Validate that variables exist in the data dictionary.
+        
+        Args:
+            variable_names (List[str]): List of variable names to validate
+            
+        Returns:
+            List[ValidationIssue]: List of validation issues found
+        """
+        issues = []
+        
+        for var_name in variable_names:
+            try:
+                # Variable validation - try multiple approaches based on the schema
+                # First try to find it as a VariableID (numeric)
+                var_exists = None
+                try:
+                    var_id = int(var_name)
+                    var_exists = self.session.execute(
+                        text("SELECT COUNT(*) FROM Variable WHERE VariableID = :var_id"),
+                        {"var_id": var_id}
+                    ).fetchone()
+                except ValueError:
+                    # Not a numeric ID, skip variable validation for now
+                    # Variables in this schema appear to be referenced by ID, not name
+                    continue
+                
+                if var_exists and var_exists[0] == 0:
+                    issue = ValidationIssue(
+                        issue_type=ValidationIssueType.MISSING_VARIABLE,
+                        description=f"Variable ID '{var_name}' not found in data dictionary",
+                        affected_element=var_name,
+                        suggested_fix=f"Add variable ID '{var_name}' to the data dictionary or check the variable ID",
+                        severity="warning",  # Changed to warning since variable structure is unclear
+                        code="VARIABLE_NOT_FOUND"
+                    )
+                    issues.append(issue)
+                    
+            except Exception as e:
+                issues.append(ValidationIssue(
+                    issue_type=ValidationIssueType.CONFIGURATION_ERROR,
+                    description=f"Error checking variable '{var_name}': {str(e)}",
+                    affected_element=var_name,
+                    severity="error",
+                    code="VARIABLE_CHECK_ERROR"
+                ))
+        
+        return issues
+    
+    def get_validation_summary(self, issues: List[ValidationIssue]) -> Dict[str, Any]:
+        """
+        Generate a summary of validation issues.
+        
+        Args:
+            issues (List[ValidationIssue]): List of validation issues
+            
+        Returns:
+            Dict[str, Any]: Summary statistics and categorized issues
+        """
+        summary = {
+            "total_issues": len(issues),
+            "by_type": {},
+            "by_severity": {},
+            "fixable_issues": [],
+            "critical_issues": []
+        }
+        
+        for issue in issues:
+            # Count by type
+            issue_type = issue.issue_type.value
+            summary["by_type"][issue_type] = summary["by_type"].get(issue_type, 0) + 1
+            
+            # Count by severity
+            summary["by_severity"][issue.severity] = summary["by_severity"].get(issue.severity, 0) + 1
+            
+            # Categorize issues
+            if issue.suggested_fix:
+                summary["fixable_issues"].append(issue)
+            
+            if issue.severity == "error":
+                summary["critical_issues"].append(issue)
+        
+        return summary
+    
+    def _parse_cell_references(self, expression: str) -> List[CellReference]:
+        """
+        Parse cell references from a DPM-XL expression.
+        
+        Args:
+            expression (str): DPM-XL expression to parse
+            
+        Returns:
+            List[CellReference]: List of parsed cell references
+        """
+        cell_refs = []
+        
+        # Regex pattern to match cell references like {tTableName, rRows, cColumns, sSheets}
+        pattern = r'\{t([^,]+),\s*([^,]+),\s*([^,]+)(?:,\s*([^}]+))?\}'
+        
+        matches = re.findall(pattern, expression)
+        
+        for match in matches:
+            table = match[0].strip()
+            
+            # Parse rows
+            rows_str = match[1].strip()
+            rows = self._parse_dimension_values(rows_str, 'r')
+            
+            # Parse columns  
+            cols_str = match[2].strip()
+            columns = self._parse_dimension_values(cols_str, 'c')
+            
+            # Parse sheets (optional)
+            sheets = []
+            if len(match) > 3 and match[3]:
+                sheets_str = match[3].strip()
+                sheets = self._parse_dimension_values(sheets_str, 's')
+            
+            cell_refs.append(CellReference(
+                table=table,
+                rows=rows,
+                columns=columns,
+                sheets=sheets
+            ))
+        
+        return cell_refs
+    
+    def _parse_dimension_values(self, dim_str: str, prefix: str) -> List[str]:
+        """
+        Parse dimension values (rows, columns, or sheets) from a string.
+        
+        Args:
+            dim_str (str): String containing dimension values
+            prefix (str): Expected prefix ('r', 'c', or 's')
+            
+        Returns:
+            List[str]: List of parsed dimension values
+        """
+        values = []
+        
+        # Remove prefix and parentheses if present
+        dim_str = dim_str.strip()
+        if dim_str.startswith(prefix):
+            dim_str = dim_str[1:]
+        if dim_str.startswith('(') and dim_str.endswith(')'):
+            dim_str = dim_str[1:-1]
+        
+        # Split by comma and clean up
+        if dim_str:
+            for value in dim_str.split(','):
+                value = value.strip()
+                if value:
+                    values.append(value)
+        
+        return values
+    
+    def __del__(self):
+        """Clean up resources."""
+        if hasattr(self, 'session'):
+            self.session.close()
+
+
+# Convenience functions for direct usage
+def validate_dpm_xl_expression(expression: str) -> List[ValidationIssue]:
+    """
+    Convenience function to validate a DPM-XL expression.
+    
+    Args:
+        expression (str): DPM-XL expression to validate
+        
+    Returns:
+        List[ValidationIssue]: List of validation issues found
+    """
+    validator = DataDictionaryValidator()
+    return validator.validate_expression_references(expression)
+
+
+def validate_table_references(table_names: List[str]) -> List[ValidationIssue]:
+    """
+    Convenience function to validate table references.
+    
+    Args:
+        table_names (List[str]): List of table names to validate
+        
+    Returns:
+        List[ValidationIssue]: List of validation issues found
+    """
+    validator = DataDictionaryValidator()
+    issues = []
+    
+    for table_name in table_names:
+        issues.extend(validator.validate_table_exists(table_name))
+    
+    return issues
+
+
+def check_data_dictionary_health() -> Dict[str, Any]:
+    """
+    Perform a comprehensive health check of the data dictionary.
+    
+    Returns:
+        Dict[str, Any]: Health check results
+    """
+    validator = DataDictionaryValidator()
+    
+    # This would include various checks like:
+    # - Missing table definitions
+    # - Orphaned references
+    # - Inconsistent naming
+    # - etc.
+    
+    health_report = {
+        "status": "healthy",  # or "warning" or "critical"
+        "total_tables": 0,
+        "issues_found": [],
+        "recommendations": []
+    }
+    
+    try:
+        # Get total table count
+        result = validator.session.execute(text("SELECT COUNT(DISTINCT table_code) FROM datapoints")).fetchone()
+        health_report["total_tables"] = result[0] if result else 0
+        
+        # Add more comprehensive checks here
+        
+    except Exception as e:
+        health_report["status"] = "critical"
+        health_report["issues_found"].append(f"Database connection error: {str(e)}")
+    
+    return health_report

py_dpm/api/migration.py

@@ -0,0 +1,77 @@
+import os
+from typing import Optional
+from sqlalchemy import Engine
+
+from py_dpm.migration import run_migration as _run_migration
+
+
+class MigrationAPI:
+    """
+    API for database migration operations.
+    
+    This class provides methods to migrate data from Access databases to SQLite.
+    """
+    
+    def __init__(self):
+        """Initialize the Migration API."""
+        pass
+    
+    def migrate_access_to_sqlite(
+        self, 
+        access_file_path: str, 
+        sqlite_db_path: Optional[str] = None
+    ) -> Engine:
+        """
+        Migrate data from an Access database to SQLite.
+        
+        Args:
+            access_file_path (str): Path to the Access database file (.mdb or .accdb)
+            sqlite_db_path (Optional[str]): Path for the SQLite database. 
+                                          If None, defaults to "database.db"
+        
+        Returns:
+            Engine: SQLAlchemy engine for the created SQLite database
+            
+        Raises:
+            FileNotFoundError: If the Access file doesn't exist
+            Exception: If migration fails
+            
+        Example:
+            >>> from pydpm.api import MigrationAPI
+            >>> migration = MigrationAPI()
+            >>> engine = migration.migrate_access_to_sqlite("data.mdb", "output.db")
+        """
+        if not os.path.exists(access_file_path):
+            raise FileNotFoundError(f"Access file not found: {access_file_path}")
+        
+        if sqlite_db_path is None:
+            sqlite_db_path = os.getenv("SQLITE_DB_PATH", "database.db")
+        
+        try:
+            engine = _run_migration(access_file_path, sqlite_db_path)
+            return engine
+        except Exception as e:
+            raise Exception(f"Migration failed: {str(e)}") from e
+
+
+# Convenience function for direct usage
+def migrate_access_to_sqlite(
+    access_file_path: str, 
+    sqlite_db_path: Optional[str] = None
+) -> Engine:
+    """
+    Convenience function to migrate Access database to SQLite.
+    
+    Args:
+        access_file_path (str): Path to the Access database file
+        sqlite_db_path (Optional[str]): Path for the SQLite database
+        
+    Returns:
+        Engine: SQLAlchemy engine for the created SQLite database
+        
+    Example:
+        >>> from pydpm.api.migration import migrate_access_to_sqlite
+        >>> engine = migrate_access_to_sqlite("data.mdb", "output.db")
+    """
+    api = MigrationAPI()
+    return api.migrate_access_to_sqlite(access_file_path, sqlite_db_path)