additory 0.1.0a2__py3-none-any.whl → 0.1.0a3__py3-none-any.whl

additory/synthetic/polars_integration.py

@@ -1,464 +0,0 @@
-"""
-Integration layer with existing polars infrastructure.
-
-Leverages existing arrow bridge components and polars infrastructure
-for enhanced performance and compatibility in synthetic data generation.
-"""
-
-import logging
-from typing import Dict, Any, Optional, Union, List
-from dataclasses import dataclass
-import polars as pl
-import pandas as pd
-
-from ..core.backends.arrow_bridge import EnhancedArrowBridge, ArrowBridgeError
-from ..core.polars_expression_engine import PolarsExpressionEngine
-from ..common.backend import detect_backend
-from .models import GenerationResult, GenerationContext
-from .exceptions import SyntheticDataError
-from .performance import performance_monitor
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass
-class IntegrationStats:
-    """Statistics for polars integration operations."""
-    arrow_conversions: int = 0
-    polars_operations: int = 0
-    memory_optimizations: int = 0
-    cross_backend_operations: int = 0
-    total_time_ms: float = 0.0
-
-
-class PolarsIntegrationLayer:
-    """
-    Integration layer that leverages existing polars infrastructure
-    for synthetic data generation operations.
-    """
-    
-    def __init__(self):
-        """Initialize the integration layer."""
-        # Use existing infrastructure components
-        self.enhanced_bridge = EnhancedArrowBridge()
-        self.expression_engine = PolarsExpressionEngine()
-        
-        # Integration statistics
-        self.stats = IntegrationStats()
-        
-        logger.info("Polars integration layer initialized")
-    
-    def optimize_dataframe_conversion(self, 
-                                    df: Union[pl.DataFrame, pd.DataFrame],
-                                    target_engine: str,
-                                    columns: Optional[List[str]] = None) -> Union[pl.DataFrame, pd.DataFrame]:
-        """
-        Optimize dataframe conversion using existing arrow bridge infrastructure.
-        
-        Args:
-            df: Source dataframe
-            target_engine: Target engine ('pandas' or 'polars')
-            columns: Optional column subset for memory efficiency
-            
-        Returns:
-            Converted dataframe in target format
-        """
-        with performance_monitor.monitor_operation(
-            "polars_integration_conversion",
-            rows=len(df),
-            columns=len(columns) if columns else len(df.columns)
-        ):
-            try:
-                source_backend = detect_backend(df)
-                
-                # Use column subsetting for memory efficiency
-                if columns and len(columns) < len(df.columns):
-                    logger.debug(f"Using column subsetting: {len(columns)}/{len(df.columns)} columns")
-                
-                # Convert using existing arrow bridge
-                arrow_table = self.enhanced_bridge.to_arrow(df, source_backend)
-                result_df = self.enhanced_bridge.from_arrow(arrow_table, target_engine)
-                
-                # Update statistics
-                self.stats.arrow_conversions += 1
-                if source_backend != target_engine:
-                    self.stats.cross_backend_operations += 1
-                
-                logger.debug(f"Converted {source_backend} to {target_engine} via Arrow bridge")
-                return result_df
-                
-            except Exception as e:
-                raise SyntheticDataError(f"Dataframe conversion failed: {e}")
-    
-    def enhance_generation_result(self, 
-                                result: GenerationResult,
-                                use_memory_optimization: bool = True) -> GenerationResult:
-        """
-        Enhance generation result using existing polars infrastructure.
-        
-        Args:
-            result: Original generation result
-            use_memory_optimization: Whether to apply memory optimizations
-            
-        Returns:
-            Enhanced generation result
-        """
-        try:
-            # Apply memory optimization if requested
-            if use_memory_optimization and hasattr(result.dataframe, 'lazy'):
-                # Use polars lazy evaluation for memory optimization
-                if isinstance(result.dataframe, pl.DataFrame):
-                    optimized_df = result.dataframe.lazy().collect()
-                    result.dataframe = optimized_df
-                    self.stats.memory_optimizations += 1
-                    logger.debug("Applied polars lazy evaluation optimization")
-            
-            # Add integration metadata
-            if 'integration_layer' not in result.metadata:
-                result.metadata['integration_layer'] = {
-                    'arrow_bridge_used': True,
-                    'polars_optimized': use_memory_optimization,
-                    'backend_detected': detect_backend(result.dataframe)
-                }
-            
-            return result
-            
-        except Exception as e:
-            logger.warning(f"Result enhancement failed: {e}")
-            return result  # Return original result if enhancement fails
-    
-    def create_cross_backend_context(self, 
-                                   context: GenerationContext,
-                                   preferred_backend: str = 'polars') -> GenerationContext:
-        """
-        Create a generation context optimized for cross-backend operations.
-        
-        Args:
-            context: Original generation context
-            preferred_backend: Preferred backend for internal operations
-            
-        Returns:
-            Optimized generation context
-        """
-        try:
-            # Clone the context
-            optimized_context = GenerationContext(
-                schema=context.schema,
-                resolved_patterns=context.resolved_patterns,
-                target_rows=context.target_rows,
-                output_engine=context.output_engine,
-                batch_size=context.batch_size,
-                seed=context.seed
-            )
-            
-            # Optimize batch size using existing infrastructure
-            if hasattr(self.enhanced_bridge, 'get_memory_stats'):
-                memory_stats = self.enhanced_bridge.get_memory_stats()
-                if memory_stats.get('cleanup_needed', False):
-                    # Reduce batch size if memory pressure is high
-                    optimized_context.batch_size = min(context.batch_size, 5000)
-                    logger.debug("Reduced batch size due to memory pressure")
-            
-            return optimized_context
-            
-        except Exception as e:
-            logger.warning(f"Context optimization failed: {e}")
-            return context  # Return original context if optimization fails
-    
-    def apply_polars_expression(self, 
-                              df: Union[pl.DataFrame, pd.DataFrame],
-                              expression: str,
-                              output_column: str) -> Union[pl.DataFrame, pd.DataFrame]:
-        """
-        Apply expression using existing polars expression engine.
-        
-        Args:
-            df: Input dataframe
-            expression: Expression to apply
-            output_column: Name for output column
-            
-        Returns:
-            Dataframe with expression result
-        """
-        try:
-            # Use existing expression engine
-            result = self.expression_engine.execute_expression(
-                df, expression, output_column
-            )
-            
-            self.stats.polars_operations += 1
-            self.stats.total_time_ms += result.execution_time_ms
-            
-            logger.debug(f"Applied expression '{expression}' in {result.execution_time_ms:.1f}ms")
-            return result.dataframe
-            
-        except Exception as e:
-            raise SyntheticDataError(f"Expression application failed: {e}")
-    
-    def optimize_memory_usage(self, 
-                            df: Union[pl.DataFrame, pd.DataFrame],
-                            aggressive: bool = False) -> Union[pl.DataFrame, pd.DataFrame]:
-        """
-        Optimize memory usage using existing infrastructure.
-        
-        Args:
-            df: Input dataframe
-            aggressive: Whether to use aggressive optimization
-            
-        Returns:
-            Memory-optimized dataframe
-        """
-        try:
-            backend = detect_backend(df)
-            
-            if backend == 'polars' and isinstance(df, pl.DataFrame):
-                # Use polars-specific optimizations
-                if aggressive:
-                    # Use lazy evaluation and collect
-                    optimized_df = df.lazy().collect()
-                else:
-                    # Simple rechunk for better memory layout
-                    optimized_df = df.rechunk()
-                
-                self.stats.memory_optimizations += 1
-                logger.debug(f"Applied polars memory optimization (aggressive={aggressive})")
-                return optimized_df
-            
-            elif backend == 'pandas':
-                # For pandas, convert to polars, optimize, then convert back
-                polars_df = pl.from_pandas(df)
-                optimized_polars = polars_df.lazy().collect() if aggressive else polars_df.rechunk()
-                optimized_df = optimized_polars.to_pandas()
-                
-                self.stats.memory_optimizations += 1
-                self.stats.cross_backend_operations += 1
-                logger.debug("Applied cross-backend memory optimization")
-                return optimized_df
-            
-            else:
-                logger.debug("No memory optimization applied for unknown backend")
-                return df
-                
-        except Exception as e:
-            logger.warning(f"Memory optimization failed: {e}")
-            return df  # Return original dataframe if optimization fails
-    
-    def validate_integration_compatibility(self, 
-                                         df: Union[pl.DataFrame, pd.DataFrame]) -> Dict[str, Any]:
-        """
-        Validate compatibility with existing polars infrastructure.
-        
-        Args:
-            df: Dataframe to validate
-            
-        Returns:
-            Compatibility report
-        """
-        try:
-            backend = detect_backend(df)
-            
-            # Test arrow conversion
-            arrow_compatible = True
-            try:
-                arrow_table, _ = to_arrow(df)
-                from_arrow(arrow_table, backend)
-            except Exception:
-                arrow_compatible = False
-            
-            # Test expression engine compatibility
-            expression_compatible = True
-            try:
-                # Simple test expression
-                self.expression_engine.validate_expression("1 + 1")
-            except Exception:
-                expression_compatible = False
-            
-            # Test enhanced bridge compatibility
-            enhanced_compatible = True
-            try:
-                self.enhanced_bridge.detect_backend(df)
-            except Exception:
-                enhanced_compatible = False
-            
-            return {
-                'backend': backend,
-                'arrow_compatible': arrow_compatible,
-                'expression_compatible': expression_compatible,
-                'enhanced_bridge_compatible': enhanced_compatible,
-                'overall_compatible': all([
-                    arrow_compatible,
-                    expression_compatible,
-                    enhanced_compatible
-                ])
-            }
-            
-        except Exception as e:
-            return {
-                'error': str(e),
-                'overall_compatible': False
-            }
-    
-    def get_integration_stats(self) -> Dict[str, Any]:
-        """Get integration layer statistics."""
-        bridge_stats = self.arrow_bridge.get_stats()
-        enhanced_stats = self.enhanced_bridge.get_conversion_stats()
-        expression_stats = self.expression_engine.get_execution_stats()
-        
-        return {
-            'integration_layer': {
-                'arrow_conversions': self.stats.arrow_conversions,
-                'polars_operations': self.stats.polars_operations,
-                'memory_optimizations': self.stats.memory_optimizations,
-                'cross_backend_operations': self.stats.cross_backend_operations,
-                'total_time_ms': self.stats.total_time_ms
-            },
-            'arrow_bridge': bridge_stats,
-            'enhanced_bridge': {
-                'conversions': enhanced_stats.conversions,
-                'memory_used_mb': enhanced_stats.memory_used_mb,
-                'total_rows_processed': enhanced_stats.total_rows_processed,
-                'cleanup_count': enhanced_stats.cleanup_count
-            },
-            'expression_engine': expression_stats
-        }
-    
-    def cleanup_integration_resources(self):
-        """Clean up integration layer resources."""
-        try:
-            # Cleanup arrow bridge cache
-            self.arrow_bridge.clear_cache()
-            
-            # Cleanup enhanced bridge memory
-            self.enhanced_bridge.cleanup_arrow_memory()
-            
-            # Reset expression engine stats if needed
-            if self.expression_engine.get_execution_stats()['total_executions'] > 1000:
-                self.expression_engine.reset_stats()
-            
-            logger.debug("Integration layer resources cleaned up")
-            
-        except Exception as e:
-            logger.warning(f"Integration cleanup failed: {e}")
-    
-    def benchmark_integration_performance(self, 
-                                        df: Union[pl.DataFrame, pd.DataFrame],
-                                        operations: List[str] = None) -> Dict[str, Any]:
-        """
-        Benchmark integration layer performance.
-        
-        Args:
-            df: Test dataframe
-            operations: List of operations to benchmark
-            
-        Returns:
-            Benchmark results
-        """
-        if operations is None:
-            operations = ['conversion', 'memory_optimization', 'expression']
-        
-        results = {}
-        
-        try:
-            # Benchmark conversion
-            if 'conversion' in operations:
-                import time
-                start_time = time.time()
-                
-                backend = detect_backend(df)
-                target = 'pandas' if backend == 'polars' else 'polars'
-                converted = self.optimize_dataframe_conversion(df, target)
-                
-                conversion_time = (time.time() - start_time) * 1000
-                results['conversion'] = {
-                    'time_ms': conversion_time,
-                    'source_backend': backend,
-                    'target_backend': target,
-                    'rows': len(df),
-                    'columns': len(df.columns)
-                }
-            
-            # Benchmark memory optimization
-            if 'memory_optimization' in operations:
-                import time
-                start_time = time.time()
-                
-                optimized = self.optimize_memory_usage(df, aggressive=True)
-                
-                optimization_time = (time.time() - start_time) * 1000
-                results['memory_optimization'] = {
-                    'time_ms': optimization_time,
-                    'backend': detect_backend(df)
-                }
-            
-            # Benchmark expression
-            if 'expression' in operations:
-                try:
-                    expr_result = self.expression_engine.benchmark_expression(
-                        df, "1 + 1", "test_column", iterations=3
-                    )
-                    results['expression'] = expr_result
-                except Exception as e:
-                    results['expression'] = {'error': str(e)}
-            
-            return results
-            
-        except Exception as e:
-            return {'error': str(e)}
-
-
-# Global integration layer instance
-_integration_layer = PolarsIntegrationLayer()
-
-
-# Convenience functions
-def optimize_conversion(df: Union[pl.DataFrame, pd.DataFrame], 
-                       target_engine: str,
-                       columns: Optional[List[str]] = None) -> Union[pl.DataFrame, pd.DataFrame]:
-    """Optimize dataframe conversion using integration layer."""
-    return _integration_layer.optimize_dataframe_conversion(df, target_engine, columns)
-
-
-def enhance_result(result: GenerationResult, 
-                  use_memory_optimization: bool = True) -> GenerationResult:
-    """Enhance generation result using integration layer."""
-    return _integration_layer.enhance_generation_result(result, use_memory_optimization)
-
-
-def optimize_context(context: GenerationContext, 
-                    preferred_backend: str = 'polars') -> GenerationContext:
-    """Optimize generation context using integration layer."""
-    return _integration_layer.create_cross_backend_context(context, preferred_backend)
-
-
-def apply_expression(df: Union[pl.DataFrame, pd.DataFrame],
-                    expression: str,
-                    output_column: str) -> Union[pl.DataFrame, pd.DataFrame]:
-    """Apply expression using integration layer."""
-    return _integration_layer.apply_polars_expression(df, expression, output_column)
-
-
-def optimize_memory(df: Union[pl.DataFrame, pd.DataFrame],
-                   aggressive: bool = False) -> Union[pl.DataFrame, pd.DataFrame]:
-    """Optimize memory usage using integration layer."""
-    return _integration_layer.optimize_memory_usage(df, aggressive)
-
-
-def validate_compatibility(df: Union[pl.DataFrame, pd.DataFrame]) -> Dict[str, Any]:
-    """Validate integration compatibility."""
-    return _integration_layer.validate_integration_compatibility(df)
-
-
-def get_integration_stats() -> Dict[str, Any]:
-    """Get integration layer statistics."""
-    return _integration_layer.get_integration_stats()
-
-
-def cleanup_integration() -> None:
-    """Clean up integration layer resources."""
-    _integration_layer.cleanup_integration_resources()
-
-
-def benchmark_integration(df: Union[pl.DataFrame, pd.DataFrame],
-                         operations: List[str] = None) -> Dict[str, Any]:
-    """Benchmark integration layer performance."""
-    return _integration_layer.benchmark_integration_performance(df, operations)

additory/synthetic/proxy.py

@@ -1,60 +0,0 @@
-import random
-import string
-import pandas as pd
-from datetime import datetime, timedelta
-
-def random_string(n=8):
-    return ''.join(random.choices(string.ascii_letters, k=n))
-
-def random_int(low=0, high=100):
-    return random.randint(low, high)
-
-def random_float(low=0, high=1):
-    return random.uniform(low, high)
-
-def random_date(start="2020-01-01", end="2023-01-01"):
-    start_dt = datetime.fromisoformat(start)
-    end_dt = datetime.fromisoformat(end)
-    delta = end_dt - start_dt
-    return start_dt + timedelta(days=random.randint(0, delta.days))
-
-def random_email():
-    return random_string(6).lower() + "@example.com"
-
-def synthetic(rows, schema):
-    """
-    schema = {
-        "name": "string",
-        "age": ("int", 18, 60),
-        "score": ("float", 0, 1),
-        "signup": ("date", "2020-01-01", "2023-01-01"),
-        "email": "email"
-    }
-    """
-
-    data = {}
-
-    for col, rule in schema.items():
-
-        if rule == "string":
-            data[col] = [random_string() for _ in range(rows)]
-
-        elif rule == "email":
-            data[col] = [random_email() for _ in range(rows)]
-
-        elif isinstance(rule, tuple) and rule[0] == "int":
-            _, low, high = rule
-            data[col] = [random_int(low, high) for _ in range(rows)]
-
-        elif isinstance(rule, tuple) and rule[0] == "float":
-            _, low, high = rule
-            data[col] = [random_float(low, high) for _ in range(rows)]
-
-        elif isinstance(rule, tuple) and rule[0] == "date":
-            _, start, end = rule
-            data[col] = [random_date(start, end) for _ in range(rows)]
-
-        else:
-            raise ValueError(f"Unknown schema rule: {rule}")
-
-    return pd.DataFrame(data)