additory 0.1.0a2__py3-none-any.whl → 0.1.0a3__py3-none-any.whl

additory/synthetic/generator.py

@@ -1,702 +0,0 @@
-"""
-Polars-native generation engine for synthetic data.
-
-Implements regex-based value generation using polars operations with
-distribution strategy integration and memory-efficient batch processing.
-"""
-
-import re
-import random
-import string
-from typing import Dict, List, Optional, Union, Any
-from dataclasses import dataclass
-import polars as pl
-import pandas as pd
-
-from .models import (
-    GenerationContext,
-    GenerationResult,
-    ResolvedPattern,
-    DistributionStrategy,
-    ValidationResult,
-    DistributionType
-)
-from .engines import DistributionManager
-from .exceptions import SyntheticDataError, ValidationError
-from .validator import ValidationSystem
-from .performance import performance_monitor, performance_optimizer, PerformanceMetrics
-from .polars_integration import optimize_conversion, enhance_result, optimize_context, optimize_memory
-
-
-@dataclass
-class GenerationConfig:
-    """Configuration for data generation operations."""
-    batch_size: int = 10000
-    seed: Optional[int] = None
-    validate_patterns: bool = True
-    memory_limit_mb: Optional[int] = None
-    enable_performance_monitoring: bool = True
-    auto_optimize_batch_size: bool = True
-    lazy_evaluation: bool = True
-    garbage_collection_frequency: int = 5
-
-
-class RegexGenerator:
-    """Generates realistic values from regex patterns."""
-    
-    def __init__(self, seed: Optional[int] = None):
-        """Initialize the regex generator with optional seed."""
-        self.seed = seed
-        self.random = random.Random(seed)
-        
-        # Regex compilation cache for performance
-        self._regex_cache: Dict[str, re.Pattern] = {}
-        self._pattern_cache: Dict[str, List[str]] = {}
-        
-    def _get_compiled_regex(self, pattern: str) -> re.Pattern:
-        """Get compiled regex from cache or compile and cache it."""
-        if pattern not in self._regex_cache:
-            try:
-                self._regex_cache[pattern] = re.compile(pattern)
-            except re.error as e:
-                raise SyntheticDataError(f"Invalid regex pattern '{pattern}': {e}")
-        return self._regex_cache[pattern]
-    
-    def warm_cache(self, patterns: List[str]) -> None:
-        """Pre-compile and cache regex patterns for better performance."""
-        for pattern in patterns:
-            self._get_compiled_regex(pattern)
-    
-    def clear_cache(self) -> None:
-        """Clear the regex compilation cache."""
-        self._regex_cache.clear()
-        self._pattern_cache.clear()
-    
-    def get_cache_stats(self) -> Dict[str, int]:
-        """Get cache statistics."""
-        return {
-            'regex_cache_size': len(self._regex_cache),
-            'pattern_cache_size': len(self._pattern_cache)
-        }
-    
-    def generate_values(self, pattern: str, count: int) -> List[str]:
-        """Generate realistic values from a regex pattern."""
-        try:
-            # For now, implement basic regex generation
-            # This is a simplified implementation - in production, you'd want
-            # a more sophisticated regex-to-value generator
-            return self._generate_from_pattern(pattern, count)
-        except Exception as e:
-            raise SyntheticDataError(f"Failed to generate values from pattern '{pattern}': {e}")
-    
-    def _generate_from_pattern(self, pattern: str, count: int) -> List[str]:
-        """Generate values from regex pattern using pattern analysis."""
-        # Remove anchors if present
-        clean_pattern = pattern.strip('^$')
-        
-        # Handle common patterns
-        if self._is_email_pattern(clean_pattern):
-            return self._generate_emails(count)
-        elif self._is_phone_pattern(clean_pattern):
-            return self._generate_phones(count)
-        elif self._is_uuid_pattern(clean_pattern):
-            return self._generate_uuids(count)
-        elif self._is_numeric_pattern(clean_pattern):
-            return self._generate_numbers(clean_pattern, count)
-        elif self._is_name_pattern(clean_pattern):
-            return self._generate_names(count)
-        else:
-            return self._generate_generic(clean_pattern, count)
-    
-    def _is_email_pattern(self, pattern: str) -> bool:
-        """Check if pattern looks like an email regex."""
-        email_indicators = ['@', r'\@', r'[A-Za-z0-9._%+-]+', r'\.[A-Za-z]{2,}']
-        return any(indicator in pattern for indicator in email_indicators)
-    
-    def _is_phone_pattern(self, pattern: str) -> bool:
-        """Check if pattern looks like a phone regex."""
-        phone_indicators = [r'\+?', r'[0-9\s\-\(\)]', r'\d{3}', r'\(\d{3}\)']
-        return any(indicator in pattern for indicator in phone_indicators)
-    
-    def _is_uuid_pattern(self, pattern: str) -> bool:
-        """Check if pattern looks like a UUID regex."""
-        uuid_indicators = [r'[0-9a-f]{8}', r'[0-9A-F]{8}', r'[a-f0-9]{4}', r'\-[a-f0-9]{4}\-']
-        return any(indicator in pattern for indicator in uuid_indicators)
-    
-    def _is_numeric_pattern(self, pattern: str) -> bool:
-        """Check if pattern is primarily numeric."""
-        numeric_indicators = [r'\d+', r'[0-9]+', r'\d{', r'[0-9]{']
-        return any(indicator in pattern for indicator in numeric_indicators)
-    
-    def _is_name_pattern(self, pattern: str) -> bool:
-        """Check if pattern looks like a name regex."""
-        name_indicators = [r'[A-Z][a-z]+', r'[A-Za-z]+\s[A-Za-z]+']
-        return any(indicator in pattern for indicator in name_indicators)
-    
-    def _generate_emails(self, count: int) -> List[str]:
-        """Generate realistic email addresses."""
-        domains = ['gmail.com', 'yahoo.com', 'hotmail.com', 'outlook.com', 'company.com', 'test.org', 'example.net']
-        prefixes = ['user', 'john', 'jane', 'test', 'admin', 'info', 'contact', 'support', 'sales', 'dev']
-        
-        emails = []
-        for i in range(count):
-            prefix = self.random.choice(prefixes)
-            number = self.random.randint(1, 9999) if self.random.random() > 0.3 else ""
-            domain = self.random.choice(domains)
-            # Add some variation with dots and underscores
-            separator = self.random.choice(['', '.', '_']) if self.random.random() > 0.5 else ""
-            emails.append(f"{prefix}{separator}{number}@{domain}")
-        
-        return emails
-    
-    def _generate_phones(self, count: int) -> List[str]:
-        """Generate realistic phone numbers."""
-        formats = [
-            lambda: f"+1-{self.random.randint(200, 999)}-{self.random.randint(200, 999)}-{self.random.randint(1000, 9999)}",
-            lambda: f"({self.random.randint(200, 999)}) {self.random.randint(200, 999)}-{self.random.randint(1000, 9999)}",
-            lambda: f"{self.random.randint(200, 999)}-{self.random.randint(200, 999)}-{self.random.randint(1000, 9999)}",
-            lambda: f"{self.random.randint(200, 999)}.{self.random.randint(200, 999)}.{self.random.randint(1000, 9999)}"
-        ]
-        
-        return [self.random.choice(formats)() for _ in range(count)]
-    
-    def _generate_uuids(self, count: int) -> List[str]:
-        """Generate realistic UUID-like strings."""
-        import uuid
-        return [str(uuid.uuid4()) for _ in range(count)]
-    
-    def _generate_numbers(self, pattern: str, count: int) -> List[str]:
-        """Generate numbers based on pattern analysis."""
-        # Extract number ranges from pattern
-        if r'\d{' in pattern:
-            # Extract digit count
-            match = re.search(r'\\d\{(\d+)\}', pattern)
-            if match:
-                digit_count = int(match.group(1))
-                min_val = 10**(digit_count-1) if digit_count > 1 else 0
-                max_val = 10**digit_count - 1
-                return [str(self.random.randint(min_val, max_val)) for _ in range(count)]
-        
-        # Default numeric generation
-        return [str(self.random.randint(1, 99999)) for _ in range(count)]
-    
-    def _generate_names(self, count: int) -> List[str]:
-        """Generate realistic names."""
-        first_names = ['John', 'Jane', 'Michael', 'Sarah', 'David', 'Lisa', 'Robert', 'Emily']
-        last_names = ['Smith', 'Johnson', 'Williams', 'Brown', 'Jones', 'Garcia', 'Miller', 'Davis']
-        
-        return [f"{self.random.choice(first_names)} {self.random.choice(last_names)}" for _ in range(count)]
-    
-    def _generate_generic(self, pattern: str, count: int) -> List[str]:
-        """Generate generic values for unrecognized patterns."""
-        # Simple fallback - generate alphanumeric strings
-        length = self._estimate_length_from_pattern(pattern)
-        
-        values = []
-        for _ in range(count):
-            value = ''.join(self.random.choices(string.ascii_letters + string.digits, k=length))
-            values.append(value)
-        
-        return values
-    
-    def _estimate_length_from_pattern(self, pattern: str) -> int:
-        """Estimate appropriate length for generated values."""
-        # Look for explicit length specifications
-        length_match = re.search(r'\{(\d+)\}', pattern)
-        if length_match:
-            return int(length_match.group(1))
-        
-        range_match = re.search(r'\{(\d+),(\d+)\}', pattern)
-        if range_match:
-            min_len, max_len = int(range_match.group(1)), int(range_match.group(2))
-            return self.random.randint(min_len, max_len)
-        
-        # Default length based on pattern complexity
-        if len(pattern) > 50:
-            return self.random.randint(8, 15)
-        elif len(pattern) > 20:
-            return self.random.randint(5, 10)
-        else:
-            return self.random.randint(3, 8)
-
-
-class PolarsGeneratorCore:
-    """Core polars-based data generation engine."""
-    
-    def __init__(self, config: Optional[GenerationConfig] = None):
-        """Initialize the polars generator core."""
-        self.config = config or GenerationConfig()
-        self.regex_generator = RegexGenerator(seed=self.config.seed)
-        self.distribution_manager = DistributionManager(seed=self.config.seed)
-        self.validation_system = ValidationSystem()
-    
-    def generate_column(self, 
-                       pattern: ResolvedPattern, 
-                       distribution: DistributionStrategy, 
-                       rows: int,
-                       column_name: str = "values") -> pl.Series:
-        """Generate a single column using polars operations."""
-        try:
-            # Step 1: Generate base values from regex pattern
-            base_values = self._generate_base_values(pattern, rows)
-            
-            # Step 2: Apply distribution strategy
-            result_series = self.distribution_manager.apply_distribution(
-                distribution, base_values, rows
-            )
-            
-            # Step 3: Rename series to column name
-            return result_series.alias(column_name)
-            
-        except Exception as e:
-            raise SyntheticDataError(f"Failed to generate column '{column_name}': {e}")
-    
-    def generate_dataframe(self, context: GenerationContext) -> pl.DataFrame:
-        """Generate complete dataframe from generation context."""
-        # Only validate context if validation is enabled
-        if self.config.validate_patterns and not context.validate_context():
-            raise ValidationError("Invalid generation context")
-        
-        try:
-            columns = []
-            
-            # Generate each column
-            for field_name, field_def in context.schema.field_definitions.items():
-                pattern = context.get_pattern(field_def.pattern_name)
-                if not pattern:
-                    raise SyntheticDataError(f"Pattern '{field_def.pattern_name}' not found")
-                
-                column = self.generate_column(
-                    pattern=pattern,
-                    distribution=field_def.distribution,
-                    rows=context.target_rows,
-                    column_name=field_name
-                )
-                columns.append(column)
-            
-            # Combine columns into dataframe
-            if not columns:
-                raise SyntheticDataError("No columns to generate")
-            
-            # Create dataframe from series
-            df_data = {col.name: col for col in columns}
-            return pl.DataFrame(df_data)
-            
-        except Exception as e:
-            raise SyntheticDataError(f"Failed to generate dataframe: {e}")
-    
-    def _generate_base_values(self, pattern: ResolvedPattern, rows: int) -> List[str]:
-        """Generate base values from a resolved pattern."""
-        # Only check pattern validity if validation is enabled
-        if self.config.validate_patterns and not pattern.is_valid:
-            raise ValidationError(f"Pattern '{pattern.name}' is not valid for generation")
-        
-        # Calculate how many unique values we need
-        # Always generate at least 10 unique values to support all distribution strategies
-        # This ensures skewed distribution (which needs >=2) and others work properly
-        unique_count = max(10, min(rows, rows // 10 + 10))
-        
-        return self.regex_generator.generate_values(pattern.regex, unique_count)
-    
-    def validate_generation_context(self, context: GenerationContext) -> ValidationResult:
-        """Validate that the generation context is ready for data generation."""
-        # Use the comprehensive validation system for fail-fast validation
-        return self.validation_system.validate_complete_configuration(
-            context.schema,
-            context.resolved_patterns,
-            context.schema.source_file
-        )
-
-
-class OutputConverter:
-    """Handles format conversion and output optimization."""
-    
-    def __init__(self):
-        """Initialize the output converter."""
-        pass
-    
-    def to_pandas(self, df: pl.DataFrame) -> pd.DataFrame:
-        """Convert polars DataFrame to pandas with optimization."""
-        import time
-        start_time = time.time()
-        
-        try:
-            # Use integration layer for optimized conversion
-            result = optimize_conversion(df, 'pandas')
-            
-            # Track conversion time
-            conversion_time_ms = (time.time() - start_time) * 1000
-            performance_monitor.track_conversion_time(conversion_time_ms)
-            
-            return result
-        except SyntheticDataError:
-            # Re-raise SyntheticDataError as-is
-            raise
-        except Exception as e:
-            # Wrap other exceptions
-            raise SyntheticDataError(f"Failed to convert to pandas: {e}")
-    
-    def to_polars(self, df: pl.DataFrame) -> pl.DataFrame:
-        """Return polars DataFrame as-is (zero-copy operation)."""
-        # Zero-copy operation - return the same object
-        performance_monitor.track_conversion_time(0.0)  # Zero-copy operation
-        return df
-    
-    def optimize_memory(self, df: pl.DataFrame) -> pl.DataFrame:
-        """Optimize memory usage of the dataframe."""
-        try:
-            # Use polars lazy evaluation for memory optimization
-            performance_monitor.track_polars_operation()
-            return df.lazy().collect()
-        except Exception as e:
-            raise SyntheticDataError(f"Failed to optimize memory: {e}")
-
-
-class SyntheticDataGenerator:
-    """High-level synthetic data generator combining all components."""
-    
-    def __init__(self, config: Optional[GenerationConfig] = None):
-        """Initialize the synthetic data generator."""
-        self.config = config or GenerationConfig()
-        self.generator_core = PolarsGeneratorCore(self.config)
-        self.output_converter = OutputConverter()
-    
-    def generate(self, context: GenerationContext) -> GenerationResult:
-        """Generate synthetic data from a generation context."""
-        import time
-        import gc
-        
-        # Optimize context using integration layer
-        optimized_context = optimize_context(context)
-        
-        # Auto-optimize batch size if enabled
-        if self.config.auto_optimize_batch_size:
-            optimal_batch_size = performance_optimizer.optimize_batch_size(
-                optimized_context.target_rows, 
-                self.config.memory_limit_mb
-            )
-            optimized_context.batch_size = optimal_batch_size
-        
-        # Start performance monitoring
-        monitor_context = None
-        if self.config.enable_performance_monitoring:
-            monitor_context = performance_monitor.monitor_operation(
-                operation_name="synthetic_data_generation",
-                rows=optimized_context.target_rows,
-                columns=len(optimized_context.schema.field_definitions),
-                batch_size=optimized_context.batch_size,
-                output_engine=optimized_context.output_engine,
-                patterns_count=len(optimized_context.resolved_patterns)
-            )
-            monitor_context.__enter__()
-        
-        start_time = time.time()
-        conversion_start_time = None
-        
-        try:
-            # Fail-fast validation - stop immediately on any validation error
-            if self.config.validate_patterns:
-                self.generator_core.validation_system.validate_fail_fast(
-                    optimized_context.schema,
-                    optimized_context.resolved_patterns,
-                    optimized_context.schema.source_file
-                )
-            
-            # Update peak memory during generation
-            if monitor_context:
-                performance_monitor.update_peak_memory()
-                performance_monitor.track_polars_operation()
-            
-            # Generate polars dataframe
-            df = self.generator_core.generate_dataframe(optimized_context)
-            
-            # Track conversion time
-            conversion_start_time = time.time()
-            
-            # Convert to requested output format
-            if optimized_context.output_engine.lower() == "pandas":
-                output_df = self.output_converter.to_pandas(df)
-            else:
-                output_df = self.output_converter.to_polars(df)
-            
-            # Track conversion time
-            if monitor_context and conversion_start_time:
-                conversion_time_ms = (time.time() - conversion_start_time) * 1000
-                performance_monitor.track_conversion_time(conversion_time_ms)
-            
-            # Calculate generation time
-            generation_time = time.time() - start_time
-            
-            # End performance monitoring to get final metrics
-            if monitor_context:
-                monitor_context.__exit__(None, None, None)
-                monitor_context = None  # Mark as completed
-            
-            # Create result with performance metrics
-            metadata = {
-                'rows_generated': optimized_context.target_rows,
-                'columns_generated': len(optimized_context.schema.field_definitions),
-                'output_engine': optimized_context.output_engine,
-                'batch_size': optimized_context.batch_size,
-                'patterns_used': list(optimized_context.resolved_patterns.keys()),
-                'generation_time_ms': generation_time * 1000
-            }
-            
-            # Add performance metrics if monitoring is enabled
-            if self.config.enable_performance_monitoring:
-                latest_metrics = performance_monitor.get_latest_metrics()
-                if latest_metrics:
-                    metadata.update({
-                        'rows_per_second': latest_metrics.rows_per_second,
-                        'memory_delta_mb': latest_metrics.memory_delta_mb,
-                        'memory_per_row_kb': latest_metrics.memory_per_row_kb,
-                        'polars_operations': latest_metrics.polars_operations,
-                        'conversion_time_ms': latest_metrics.conversion_time_ms
-                    })
-            
-            result = GenerationResult(
-                dataframe=output_df,
-                generation_time=generation_time,
-                metadata=metadata
-            )
-            
-            # Enhance result using integration layer
-            enhanced_result = enhance_result(result, use_memory_optimization=self.config.lazy_evaluation)
-            
-            return enhanced_result
-            
-        except Exception as e:
-            raise SyntheticDataError(f"Data generation failed: {e}")
-        finally:
-            # End performance monitoring if not already done
-            if monitor_context:
-                monitor_context.__exit__(None, None, None)
-    
-    def generate_batch(self, context: GenerationContext, batch_size: Optional[int] = None) -> GenerationResult:
-        """Generate data in batches for memory efficiency."""
-        import gc
-        
-        # Auto-optimize batch size if enabled
-        if self.config.auto_optimize_batch_size and batch_size is None:
-            batch_size = performance_optimizer.optimize_batch_size(
-                context.target_rows, 
-                self.config.memory_limit_mb
-            )
-        else:
-            batch_size = batch_size or self.config.batch_size
-        
-        # Start performance monitoring for batch operation
-        monitor_context = None
-        if self.config.enable_performance_monitoring:
-            monitor_context = performance_monitor.monitor_operation(
-                operation_name="batch_synthetic_data_generation",
-                rows=context.target_rows,
-                columns=len(context.schema.field_definitions),
-                batch_size=batch_size,
-                output_engine=context.output_engine,
-                patterns_count=len(context.resolved_patterns)
-            )
-            monitor_context.__enter__()
-        
-        try:
-            if context.target_rows <= batch_size:
-                # Single batch - use regular generate method
-                result = self.generate(context)
-                if monitor_context:
-                    # Update batch count in metadata
-                    result.metadata['batch_count'] = 1
-                return result
-            
-            # Multi-batch generation
-            batches = []
-            remaining_rows = context.target_rows
-            batch_count = 0
-            
-            while remaining_rows > 0:
-                current_batch_size = min(batch_size, remaining_rows)
-                batch_count += 1
-                
-                # Create batch context
-                batch_context = GenerationContext(
-                    schema=context.schema,
-                    resolved_patterns=context.resolved_patterns,
-                    target_rows=current_batch_size,
-                    output_engine=context.output_engine,
-                    batch_size=current_batch_size,
-                    seed=context.seed
-                )
-                
-                # Generate batch with individual monitoring disabled to avoid nested monitoring
-                batch_config = GenerationConfig(
-                    batch_size=current_batch_size,
-                    seed=self.config.seed,
-                    validate_patterns=self.config.validate_patterns,
-                    memory_limit_mb=self.config.memory_limit_mb,
-                    enable_performance_monitoring=False,  # Disable for individual batches
-                    auto_optimize_batch_size=False,
-                    lazy_evaluation=self.config.lazy_evaluation,
-                    garbage_collection_frequency=self.config.garbage_collection_frequency
-                )
-                
-                batch_generator = SyntheticDataGenerator(batch_config)
-                batch_result = batch_generator.generate(batch_context)
-                batches.append(batch_result.dataframe)
-                
-                remaining_rows -= current_batch_size
-                
-                # Update peak memory and perform garbage collection if needed
-                if monitor_context:
-                    performance_monitor.update_peak_memory()
-                    performance_monitor.track_polars_operation()
-                    
-                    if batch_count % self.config.garbage_collection_frequency == 0:
-                        gc.collect()
-            
-            # Combine batches
-            if context.output_engine.lower() == "pandas":
-                combined_df = pd.concat(batches, ignore_index=True)
-            else:
-                combined_df = pl.concat(batches)
-            
-            # Create result with batch metadata
-            metadata = {
-                'rows_generated': context.target_rows,
-                'columns_generated': len(context.schema.field_definitions),
-                'output_engine': context.output_engine,
-                'batch_count': batch_count,
-                'batch_size': batch_size,
-                'total_batches': len(batches)
-            }
-            
-            # Add performance metrics if monitoring is enabled
-            if self.config.enable_performance_monitoring:
-                latest_metrics = performance_monitor.get_latest_metrics()
-                if latest_metrics:
-                    metadata.update({
-                        'rows_per_second': latest_metrics.rows_per_second,
-                        'memory_delta_mb': latest_metrics.memory_delta_mb,
-                        'memory_per_row_kb': latest_metrics.memory_per_row_kb,
-                        'polars_operations': latest_metrics.polars_operations,
-                        'peak_memory_mb': latest_metrics.memory_peak_mb
-                    })
-            
-            return GenerationResult(
-                dataframe=combined_df,
-                metadata=metadata
-            )
-            
-        except Exception as e:
-            raise SyntheticDataError(f"Batch data generation failed: {e}")
-        finally:
-            # End performance monitoring
-            if monitor_context:
-                monitor_context.__exit__(None, None, None)
-    
-    def get_performance_metrics(self) -> Optional[PerformanceMetrics]:
-        """Get the latest performance metrics."""
-        return performance_monitor.get_latest_metrics()
-    
-    def get_performance_summary(self) -> Dict[str, Any]:
-        """Get a summary of all performance metrics."""
-        return performance_monitor.get_metrics_summary()
-    
-    def get_optimization_recommendations(self) -> List[str]:
-        """Get performance optimization recommendations."""
-        return performance_monitor.get_optimization_recommendations()
-    
-    def compare_engine_performance(self, context: GenerationContext) -> Dict[str, Any]:
-        """
-        Compare performance between polars and pandas engines.
-        
-        Args:
-            context: Generation context for comparison
-            
-        Returns:
-            Dictionary with performance comparison results
-        """
-        if not self.config.enable_performance_monitoring:
-            return {"error": "Performance monitoring is disabled"}
-        
-        # Generate with polars engine
-        polars_context = GenerationContext(
-            schema=context.schema,
-            resolved_patterns=context.resolved_patterns,
-            target_rows=context.target_rows,
-            output_engine="polars",
-            batch_size=context.batch_size,
-            seed=context.seed
-        )
-        
-        polars_result = self.generate(polars_context)
-        polars_metrics = performance_monitor.get_latest_metrics()
-        
-        # Generate with pandas engine
-        pandas_context = GenerationContext(
-            schema=context.schema,
-            resolved_patterns=context.resolved_patterns,
-            target_rows=context.target_rows,
-            output_engine="pandas",
-            batch_size=context.batch_size,
-            seed=context.seed
-        )
-        
-        pandas_result = self.generate(pandas_context)
-        pandas_metrics = performance_monitor.get_latest_metrics()
-        
-        # Create comparison
-        if polars_metrics and pandas_metrics:
-            comparison = performance_monitor.compare_engines(polars_metrics, pandas_metrics)
-            return {
-                "polars_performance": {
-                    "duration_ms": polars_metrics.duration_ms,
-                    "memory_delta_mb": polars_metrics.memory_delta_mb,
-                    "rows_per_second": polars_metrics.rows_per_second,
-                    "conversion_time_ms": polars_metrics.conversion_time_ms
-                },
-                "pandas_performance": {
-                    "duration_ms": pandas_metrics.duration_ms,
-                    "memory_delta_mb": pandas_metrics.memory_delta_mb,
-                    "rows_per_second": pandas_metrics.rows_per_second,
-                    "conversion_time_ms": pandas_metrics.conversion_time_ms
-                },
-                "polars_advantage_speed": comparison.polars_advantage_speed,
-                "polars_advantage_memory": comparison.polars_advantage_memory,
-                "recommendation": comparison.recommendation
-            }
-        else:
-            return {"error": "Failed to collect performance metrics for comparison"}
-    
-    def optimize_configuration(self, target_rows: int, columns: int) -> GenerationConfig:
-        """
-        Get optimized configuration for the given generation parameters.
-        
-        Args:
-            target_rows: Number of rows to generate
-            columns: Number of columns to generate
-            
-        Returns:
-            Optimized GenerationConfig
-        """
-        # Get memory optimization config
-        memory_config = performance_optimizer.get_memory_optimization_config()
-        
-        # Determine if streaming should be used
-        use_streaming = performance_optimizer.should_use_streaming(target_rows, columns)
-        
-        # Create optimized config
-        optimized_config = GenerationConfig(
-            batch_size=memory_config["batch_size"],
-            seed=self.config.seed,
-            validate_patterns=self.config.validate_patterns,
-            memory_limit_mb=memory_config["memory_limit_mb"],
-            enable_performance_monitoring=True,
-            auto_optimize_batch_size=True,
-            lazy_evaluation=memory_config["lazy_evaluation"],
-            garbage_collection_frequency=memory_config["garbage_collection_frequency"]
-        )
-        
-        return optimized_config
-    
-    def clear_performance_history(self):
-        """Clear performance monitoring history."""
-        performance_monitor.clear_history()