additory 0.1.0a1__py3-none-any.whl

additory/expressions/proxy.py

@@ -0,0 +1,546 @@
+#!/usr/bin/env python3
+"""
+Enhanced Expression Proxy for the Enhanced Expressions System
+Integrates version management, caching, integrity validation, and Polars processing
+"""
+
+import os
+import pandas as pd
+import polars as pl
+from typing import Optional, Union, Dict, Any, List
+from dataclasses import dataclass
+
+from ..core.enhanced_version_manager import EnhancedVersionManager
+from ..core.enhanced_cache_manager import get_cache_manager
+from ..core.polars_expression_engine import PolarsExpressionEngine
+from ..core.namespace_manager import NamespaceManager
+from ..core.sample_data_manager import get_sample_data_manager, SampleDataManager
+from ..core.parser import parse_expression
+from ..core.logging import log_info, log_warning
+
+
+@dataclass
+class SampleDataRequest:
+    """Marker class for sample data requests"""
+    sample_type: str  # "clean" or "unclean"
+    
+    def __repr__(self):
+        return f"SampleDataRequest(type='{self.sample_type}')"
+
+
+# Global sample data request instances
+sample_data = SampleDataRequest("clean")
+sample_data_unclean = SampleDataRequest("unclean")
+
+
+class ExpressionNotFoundError(Exception):
+    """Raised when an expression cannot be found"""
+    pass
+
+
+class ExpressionExecutionError(Exception):
+    """Raised when expression execution fails"""
+    pass
+
+
+class EnhancedExpressionCallable:
+    """
+    Callable object representing a specific expression
+    Handles execution, sample data, and version resolution
+    """
+    
+    def __init__(self, expression_name: str, namespace: str, proxy: 'EnhancedExpressionProxy'):
+        self.expression_name = expression_name
+        self.namespace = namespace
+        self.proxy = proxy
+    
+    def __call__(self, df_or_request, version: Optional[str] = None, 
+                 backend: Optional[str] = None, insert_at: Optional[str] = None, **kwargs):
+        """
+        Execute expression or return sample data
+        
+        Args:
+            df_or_request: DataFrame to process or SampleDataRequest
+            version: Specific version to use (optional)
+            backend: Backend hint (ignored - always uses Polars)
+            insert_at: Output column name (defaults to expression name)
+            **kwargs: Additional execution parameters
+            
+        Returns:
+            DataFrame with expression results or sample data
+        """
+        try:
+            # Handle sample data requests
+            if isinstance(df_or_request, SampleDataRequest):
+                return self.proxy._handle_sample_request(
+                    self.expression_name, df_or_request.sample_type, version
+                )
+            
+            # Handle normal expression execution
+            return self.proxy._execute_expression(
+                self.expression_name, df_or_request, version, insert_at, **kwargs
+            )
+            
+        except Exception as e:
+            log_warning(f"[expression_proxy] Failed to execute {self.expression_name}: {e}")
+            raise ExpressionExecutionError(f"Failed to execute {self.expression_name}: {e}")
+    
+    def info(self, version: Optional[str] = None) -> Dict[str, Any]:
+        """Get detailed information about this expression"""
+        return self.proxy.get_expression_info(self.expression_name, version)
+    
+    def sample(self, clean: bool = True) -> pd.DataFrame:
+        """Get sample data for this expression"""
+        sample_type = "clean" if clean else "unclean"
+        return self.proxy._handle_sample_request(self.expression_name, sample_type)
+    
+    def refresh(self, version: Optional[str] = None) -> bool:
+        """Force refresh this expression from cache"""
+        resolved_version = version or self.proxy.version_manager.default_version
+        return self.proxy.cache_manager.force_refresh_expression(
+            self.expression_name, resolved_version, self.namespace
+        )
+
+
+class EnhancedExpressionProxy:
+    """
+    Enhanced expression proxy with full pipeline integration
+    Supports dual namespaces, caching, version management, and Polars processing
+    """
+    
+    def __init__(self, namespace: str = "builtin"):
+        """
+        Initialize enhanced expression proxy
+        
+        Args:
+            namespace: Namespace to operate in ("builtin" or "user")
+        """
+        self.namespace = namespace
+        
+        # Initialize core components
+        self.version_manager = EnhancedVersionManager()
+        self.cache_manager = get_cache_manager()
+        self.polars_engine = PolarsExpressionEngine()
+        self.namespace_manager = NamespaceManager()
+        self.sample_data_manager = get_sample_data_manager()
+        
+        # Configuration
+        self.default_version = None  # Use version manager's default
+        self.auto_cache = True
+        self.validate_integrity = True
+        
+        log_info(f"[expression_proxy] Initialized for namespace: {namespace}")
+    
+    def __getattr__(self, expression_name: str) -> EnhancedExpressionCallable:
+        """
+        Get expression callable for dynamic access
+        
+        Args:
+            expression_name: Name of the expression
+            
+        Returns:
+            EnhancedExpressionCallable for the expression
+        """
+        return EnhancedExpressionCallable(expression_name, self.namespace, self)
+    
+    def _resolve_expression(self, expression_name: str, version: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Resolve expression from cache or source
+        
+        Args:
+            expression_name: Name of the expression
+            version: Specific version (optional)
+            
+        Returns:
+            Dictionary with expression data and metadata
+            
+        Raises:
+            ExpressionNotFoundError: If expression cannot be found
+        """
+        try:
+            # Determine version to use
+            resolved_version = version or self.default_version or self.version_manager.default_version
+            
+            # Try to get from cache first
+            cached_path = self.cache_manager.get_cached_expression(
+                expression_name, resolved_version, self.namespace
+            )
+            
+            if cached_path:
+                log_info(f"[expression_proxy] Using cached expression: {expression_name} v{resolved_version}")
+                return self._load_expression_from_file(cached_path, expression_name, resolved_version)
+            
+            # Not in cache - try to load and cache from source
+            if self.auto_cache:
+                source_path = self._find_source_expression(expression_name, resolved_version)
+                if source_path:
+                    # Cache the expression
+                    success = self.cache_manager.cache_expression(
+                        source_path, self.namespace, expression_name, resolved_version
+                    )
+                    
+                    if success:
+                        cached_path = self.cache_manager.get_cached_expression(
+                            expression_name, resolved_version, self.namespace
+                        )
+                        if cached_path:
+                            return self._load_expression_from_file(cached_path, expression_name, resolved_version)
+                    
+                    # Fallback to direct loading
+                    return self._load_expression_from_file(source_path, expression_name, resolved_version)
+            
+            raise ExpressionNotFoundError(
+                f"Expression '{expression_name}' version '{resolved_version}' not found in namespace '{self.namespace}'"
+            )
+            
+        except Exception as e:
+            if isinstance(e, ExpressionNotFoundError):
+                raise
+            log_warning(f"[expression_proxy] Failed to resolve expression {expression_name}: {e}")
+            raise ExpressionNotFoundError(f"Failed to resolve expression {expression_name}: {e}")
+    
+    def _find_source_expression(self, expression_name: str, version: str) -> Optional[str]:
+        """Find source expression file"""
+        try:
+            # Get namespace path
+            if self.namespace == "builtin":
+                base_path = self.namespace_manager.builtin_path
+            else:
+                base_path = self.namespace_manager.user_path
+            
+            # Try to load manifest to get exact filename
+            try:
+                manifest = self.version_manager.load_manifest(base_path)
+                if version in manifest and expression_name in manifest[version].expressions:
+                    filename = manifest[version].expressions[expression_name]
+                    source_path = os.path.join(base_path, filename)
+                    if os.path.exists(source_path):
+                        return source_path
+            except Exception:
+                pass
+            
+            # Fallback: try standard filename pattern
+            filename = f"{expression_name}_{version}.add"
+            source_path = os.path.join(base_path, filename)
+            if os.path.exists(source_path):
+                return source_path
+            
+            return None
+            
+        except Exception as e:
+            log_warning(f"[expression_proxy] Failed to find source for {expression_name}: {e}")
+            return None
+    
+    def _load_expression_from_file(self, file_path: str, expression_name: str, version: str) -> Dict[str, Any]:
+        """Load and parse expression from file"""
+        try:
+            # Read file content
+            with open(file_path, 'r', encoding='utf-8') as f:
+                file_content = f.read()
+            
+            # Parse the expression file
+            parsed_data = parse_expression(file_content)
+            
+            return {
+                "name": expression_name,
+                "version": version,
+                "file_path": file_path,
+                "parsed_data": parsed_data,
+                "ast": parsed_data.ast,
+                "expression": parsed_data.expression,
+                "sample_clean": parsed_data.sample_clean,
+                "sample_unclean": parsed_data.sample_unclean,
+                "metadata": parsed_data.metadata
+            }
+            
+        except Exception as e:
+            log_warning(f"[expression_proxy] Failed to load expression from {file_path}: {e}")
+            raise ExpressionNotFoundError(f"Failed to load expression from {file_path}: {e}")
+    
+    def _execute_expression(self, expression_name: str, df: Union[pd.DataFrame, pl.DataFrame], 
+                          version: Optional[str] = None, output_column: Optional[str] = None, 
+                          **kwargs) -> Union[pd.DataFrame, pl.DataFrame]:
+        """
+        Execute expression on dataframe
+        
+        Args:
+            expression_name: Name of the expression
+            df: Input dataframe
+            version: Specific version (optional)
+            output_column: Output column name (defaults to expression name)
+            **kwargs: Additional execution parameters
+            
+        Returns:
+            DataFrame with expression results
+        """
+        try:
+            # Resolve expression
+            expression_data = self._resolve_expression(expression_name, version)
+            
+            # Determine output column
+            output_col = output_column or expression_name
+            
+            # Detect input backend
+            if isinstance(df, pd.DataFrame):
+                backend_type = "pandas"
+            elif isinstance(df, pl.DataFrame):
+                backend_type = "polars"
+            else:
+                # Try to detect other types
+                backend_type = self.polars_engine.arrow_bridge.detect_backend(df)
+            
+            # Execute using Polars engine
+            result = self.polars_engine.execute_expression(
+                df, expression_data["expression"], output_col, backend_type
+            )
+            
+            # Extract DataFrame from ExpressionResult if needed
+            if hasattr(result, 'dataframe'):
+                result = result.dataframe
+            
+            log_info(f"[expression_proxy] Executed {expression_name} v{expression_data['version']}")
+            return result
+            
+        except Exception as e:
+            log_warning(f"[expression_proxy] Expression execution failed: {e}")
+            raise ExpressionExecutionError(f"Expression execution failed: {e}")
+    
+    def _handle_sample_request(self, expression_name: str, sample_type: str, 
+                             version: Optional[str] = None) -> pd.DataFrame:
+        """
+        Handle sample data requests using enhanced sample data manager
+        
+        Args:
+            expression_name: Name of the expression
+            sample_type: "clean" or "unclean"
+            version: Specific version (optional)
+            
+        Returns:
+            DataFrame with sample data
+        """
+        try:
+            # Use enhanced sample data manager
+            if sample_type == "clean":
+                return self.sample_data_manager.get_clean_sample(
+                    expression_name, self.namespace, version
+                )
+            else:
+                return self.sample_data_manager.get_unclean_sample(
+                    expression_name, self.namespace, version
+                )
+                
+        except Exception as e:
+            log_warning(f"[expression_proxy] Failed to get sample data for {expression_name}: {e}")
+            return pd.DataFrame({
+                "error": [f"Failed to get {sample_type} sample data: {e}"]
+            })
+    
+    def get_expression_info(self, expression_name: str, version: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Get detailed information about an expression
+        
+        Args:
+            expression_name: Name of the expression
+            version: Specific version (optional)
+            
+        Returns:
+            Dictionary with expression information
+        """
+        try:
+            resolved_version = version or self.default_version or self.version_manager.default_version
+            
+            # Get cache info
+            cache_info = self.cache_manager.get_expression_cache_info(
+                expression_name, resolved_version, self.namespace
+            )
+            
+            # Get expression data
+            try:
+                expression_data = self._resolve_expression(expression_name, version)
+                metadata = expression_data.get("metadata", {})
+            except Exception:
+                metadata = {}
+            
+            return {
+                "name": expression_name,
+                "version": resolved_version,
+                "namespace": self.namespace,
+                "metadata": metadata,
+                "cache_info": cache_info,
+                "has_clean_sample": bool(expression_data.get("sample_clean")) if 'expression_data' in locals() else False,
+                "has_unclean_sample": bool(expression_data.get("sample_unclean")) if 'expression_data' in locals() else False
+            }
+            
+        except Exception as e:
+            log_warning(f"[expression_proxy] Failed to get info for {expression_name}: {e}")
+            return {
+                "name": expression_name,
+                "version": resolved_version,
+                "namespace": self.namespace,
+                "error": str(e)
+            }
+    
+    def list_expressions(self, version: Optional[str] = None) -> Dict[str, Any]:
+        """
+        List all available expressions in this namespace
+        
+        Args:
+            version: Specific version (optional)
+            
+        Returns:
+            Dictionary with expression list and metadata
+        """
+        try:
+            resolved_version = version or self.default_version or self.version_manager.default_version
+            
+            # Get namespace path
+            if self.namespace == "builtin":
+                base_path = self.namespace_manager.builtin_path
+            else:
+                base_path = self.namespace_manager.user_path
+            
+            # Try to load manifest
+            expressions = {}
+            try:
+                manifest = self.version_manager.load_manifest(base_path)
+                if resolved_version in manifest:
+                    expressions = manifest[resolved_version].expressions
+            except Exception:
+                # Fallback: scan directory for .add files
+                if os.path.exists(base_path):
+                    for filename in os.listdir(base_path):
+                        if filename.endswith(f"_{resolved_version}.add"):
+                            expr_name = filename.replace(f"_{resolved_version}.add", "")
+                            expressions[expr_name] = filename
+            
+            return {
+                "namespace": self.namespace,
+                "version": resolved_version,
+                "expressions": expressions,
+                "count": len(expressions)
+            }
+            
+        except Exception as e:
+            log_warning(f"[expression_proxy] Failed to list expressions: {e}")
+            return {
+                "namespace": self.namespace,
+                "version": resolved_version,
+                "expressions": {},
+                "count": 0,
+                "error": str(e)
+            }
+    
+    def set_default_version(self, version: str):
+        """Set default version for this proxy"""
+        self.default_version = version
+        log_info(f"[expression_proxy] Set default version to {version} for {self.namespace}")
+    
+    def refresh_cache(self) -> Dict[str, int]:
+        """Refresh cache for this namespace"""
+        return self.cache_manager.refresh_cache(self.namespace)
+    
+    def get_cache_status(self) -> Dict[str, Any]:
+        """Get cache status for this namespace"""
+        full_status = self.cache_manager.get_cache_status()
+        return {
+            "namespace": self.namespace,
+            "health": full_status.get("health", "unknown"),
+            "namespace_health": full_status.get("namespace_health", {}).get(self.namespace, {}),
+            "global_stats": full_status.get("global_stats", {})
+        }
+    
+    def get_sample_info(self, expression_name: str, version: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Get comprehensive sample data information
+        
+        Args:
+            expression_name: Name of the expression
+            version: Specific version (optional)
+            
+        Returns:
+            Dictionary with sample data information
+        """
+        try:
+            sample_info = self.sample_data_manager.get_sample_info(
+                expression_name, self.namespace, version
+            )
+            
+            return {
+                "expression_name": sample_info.expression_name,
+                "version": sample_info.version,
+                "has_clean_sample": sample_info.has_clean,
+                "has_unclean_sample": sample_info.has_unclean,
+                "clean_rows": sample_info.clean_rows,
+                "unclean_rows": sample_info.unclean_rows,
+                "educational_comments": sample_info.educational_comments,
+                "validation_errors": sample_info.validation_errors,
+                "namespace": self.namespace
+            }
+            
+        except Exception as e:
+            log_warning(f"[expression_proxy] Failed to get sample info for {expression_name}: {e}")
+            return {"error": str(e)}
+    
+    def validate_sample_data(self, expression_name: str, sample_data: Dict[str, Any], 
+                           sample_type: str = "clean", version: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Validate sample data format and content
+        
+        Args:
+            expression_name: Name of the expression
+            sample_data: Sample data to validate
+            sample_type: "clean" or "unclean"
+            version: Specific version (optional)
+            
+        Returns:
+            Dictionary with validation results
+        """
+        try:
+            is_valid, errors = self.sample_data_manager.validate_sample_data(
+                sample_data, expression_name, sample_type
+            )
+            
+            return {
+                "is_valid": is_valid,
+                "errors": errors,
+                "sample_type": sample_type,
+                "expression_name": expression_name,
+                "namespace": self.namespace
+            }
+            
+        except Exception as e:
+            log_warning(f"[expression_proxy] Failed to validate sample data for {expression_name}: {e}")
+            return {"error": str(e), "is_valid": False}
+    
+    def create_sample_template(self, expression_name: str, columns: List[str]) -> Dict[str, Any]:
+        """
+        Create sample data template for an expression
+        
+        Args:
+            expression_name: Name of the expression
+            columns: List of required columns
+            
+        Returns:
+            Dictionary with sample data templates
+        """
+        try:
+            template = self.sample_data_manager.create_sample_template(expression_name, columns)
+            
+            return {
+                "expression_name": expression_name,
+                "namespace": self.namespace,
+                "template": template,
+                "usage": {
+                    "clean": "Use for normal testing and validation",
+                    "unclean": "Use for error handling and edge case testing"
+                }
+            }
+            
+        except Exception as e:
+            log_warning(f"[expression_proxy] Failed to create sample template for {expression_name}: {e}")
+            return {"error": str(e)}
+
+
+# Backward compatibility aliases
+ExpressionProxy = EnhancedExpressionProxy
+ExpressionCallable = EnhancedExpressionCallable