additory 0.1.0a1__py3-none-any.whl

additory/expressions/samples.py

@@ -0,0 +1,492 @@
+# sample_data_manager.py
+# Enhanced sample data management for additory expressions
+
+import os
+import yaml
+import pandas as pd
+from typing import Dict, List, Optional, Any, Union, Tuple
+from dataclasses import dataclass
+import re
+
+from .logging import log_info, log_warning
+from .enhanced_version_manager import EnhancedVersionManager
+from .namespace_manager import NamespaceManager
+from .integrity_manager import IntegrityManager
+
+
+@dataclass
+class SampleDataInfo:
+    """Information about sample data"""
+    expression_name: str
+    version: str
+    has_clean: bool
+    has_unclean: bool
+    clean_rows: int
+    unclean_rows: int
+    educational_comments: List[str]
+    validation_errors: List[str]
+
+
+class SampleDataError(Exception):
+    """Raised when sample data operations fail"""
+    pass
+
+
+class SampleDataManager:
+    """
+    Enhanced sample data management system
+    Provides clean/unclean sample support with educational comments and validation
+    """
+    
+    def __init__(self):
+        self.version_manager = EnhancedVersionManager()
+        self.namespace_manager = NamespaceManager()
+        self.integrity_manager = IntegrityManager()
+        
+        # Sample data validation rules
+        self.validation_rules = {
+            "max_rows": 100,  # Maximum rows in sample data
+            "min_rows": 1,    # Minimum rows in sample data
+            "required_columns": [],  # Will be determined from expression
+            "max_column_length": 50,  # Maximum string length in columns
+        }
+        
+        # Educational comment patterns for unclean data
+        self.educational_patterns = {
+            "missing_values": "# Missing values to test null handling",
+            "invalid_types": "# Invalid data types to test type validation", 
+            "edge_cases": "# Edge cases to test boundary conditions",
+            "malformed_data": "# Malformed data to test error handling",
+            "duplicate_values": "# Duplicate values to test deduplication",
+            "extreme_values": "# Extreme values to test range validation"
+        }
+        
+        log_info("[sample_data] Sample Data Manager initialized")
+    
+    def get_clean_sample(self, expression_name: str, namespace: str = "builtin", 
+                        version: Optional[str] = None) -> pd.DataFrame:
+        """
+        Get clean sample data for an expression
+        
+        Args:
+            expression_name: Name of the expression
+            namespace: Namespace ("builtin" or "user")
+            version: Specific version (optional)
+            
+        Returns:
+            DataFrame with clean sample data
+            
+        Raises:
+            SampleDataError: If sample data cannot be retrieved
+        """
+        try:
+            sample_data = self._get_sample_data(expression_name, namespace, version, "clean")
+            
+            if sample_data is None:
+                # Generate default clean sample if none exists
+                return self._generate_default_clean_sample(expression_name)
+            
+            df = pd.DataFrame(sample_data)
+            
+            # Validate clean sample data
+            validation_errors = self._validate_clean_sample(df, expression_name)
+            if validation_errors:
+                log_warning(f"[sample_data] Clean sample validation issues for {expression_name}: {validation_errors}")
+            
+            log_info(f"[sample_data] Retrieved clean sample for {expression_name} ({len(df)} rows)")
+            return df
+            
+        except Exception as e:
+            log_warning(f"[sample_data] Failed to get clean sample for {expression_name}: {e}")
+            raise SampleDataError(f"Failed to get clean sample data: {e}")
+    
+    def get_unclean_sample(self, expression_name: str, namespace: str = "builtin", 
+                          version: Optional[str] = None) -> pd.DataFrame:
+        """
+        Get unclean sample data with educational comments
+        
+        Args:
+            expression_name: Name of the expression
+            namespace: Namespace ("builtin" or "user")
+            version: Specific version (optional)
+            
+        Returns:
+            DataFrame with unclean sample data and educational comments
+            
+        Raises:
+            SampleDataError: If sample data cannot be retrieved
+        """
+        try:
+            sample_data = self._get_sample_data(expression_name, namespace, version, "unclean")
+            
+            if sample_data is None:
+                # Generate default unclean sample if none exists
+                return self._generate_default_unclean_sample(expression_name)
+            
+            df = pd.DataFrame(sample_data)
+            
+            # Add educational comments as metadata
+            df = self._add_educational_comments(df, expression_name)
+            
+            log_info(f"[sample_data] Retrieved unclean sample for {expression_name} ({len(df)} rows)")
+            return df
+            
+        except Exception as e:
+            log_warning(f"[sample_data] Failed to get unclean sample for {expression_name}: {e}")
+            raise SampleDataError(f"Failed to get unclean sample data: {e}")
+    
+    def validate_sample_data(self, sample_data: Dict[str, Any], 
+                           expression_name: str, sample_type: str = "clean") -> Tuple[bool, List[str]]:
+        """
+        Validate sample data format and content
+        
+        Args:
+            sample_data: Sample data dictionary
+            expression_name: Name of the expression
+            sample_type: "clean" or "unclean"
+            
+        Returns:
+            Tuple of (is_valid, list_of_errors)
+        """
+        errors = []
+        
+        try:
+            # Check if sample_data is a dictionary
+            if not isinstance(sample_data, dict):
+                errors.append("Sample data must be a dictionary")
+                return False, errors
+            
+            # Check if sample_data has columns
+            if not sample_data:
+                errors.append("Sample data cannot be empty")
+                return False, errors
+            
+            # Convert to DataFrame for validation
+            try:
+                df = pd.DataFrame(sample_data)
+            except Exception as e:
+                errors.append(f"Cannot convert sample data to DataFrame: {e}")
+                return False, errors
+            
+            # Validate row count
+            row_count = len(df)
+            if row_count < self.validation_rules["min_rows"]:
+                errors.append(f"Sample data has too few rows: {row_count} < {self.validation_rules['min_rows']}")
+            
+            if row_count > self.validation_rules["max_rows"]:
+                errors.append(f"Sample data has too many rows: {row_count} > {self.validation_rules['max_rows']}")
+            
+            # Validate column content
+            for column, values in sample_data.items():
+                if not isinstance(values, list):
+                    errors.append(f"Column '{column}' must be a list")
+                    continue
+                
+                # Check for consistent length
+                if len(values) != row_count:
+                    errors.append(f"Column '{column}' has inconsistent length")
+                
+                # Check string length limits
+                for i, value in enumerate(values):
+                    if isinstance(value, str) and len(value) > self.validation_rules["max_column_length"]:
+                        errors.append(f"Column '{column}' row {i} exceeds max length")
+            
+            # Specific validation for clean vs unclean samples
+            if sample_type == "clean":
+                errors.extend(self._validate_clean_sample(df, expression_name))
+            else:
+                errors.extend(self._validate_unclean_sample(df, expression_name))
+            
+            is_valid = len(errors) == 0
+            
+            if is_valid:
+                log_info(f"[sample_data] Sample data validation passed for {expression_name}")
+            else:
+                log_warning(f"[sample_data] Sample data validation failed for {expression_name}: {errors}")
+            
+            return is_valid, errors
+            
+        except Exception as e:
+            errors.append(f"Validation error: {e}")
+            return False, errors
+    
+    def get_sample_info(self, expression_name: str, namespace: str = "builtin", 
+                       version: Optional[str] = None) -> SampleDataInfo:
+        """
+        Get comprehensive information about sample data
+        
+        Args:
+            expression_name: Name of the expression
+            namespace: Namespace ("builtin" or "user")
+            version: Specific version (optional)
+            
+        Returns:
+            SampleDataInfo object with comprehensive information
+        """
+        try:
+            # Get sample data
+            clean_data = self._get_sample_data(expression_name, namespace, version, "clean")
+            unclean_data = self._get_sample_data(expression_name, namespace, version, "unclean")
+            
+            # Analyze clean sample
+            has_clean = clean_data is not None
+            clean_rows = len(pd.DataFrame(clean_data)) if has_clean else 0
+            
+            # Analyze unclean sample
+            has_unclean = unclean_data is not None
+            unclean_rows = len(pd.DataFrame(unclean_data)) if has_unclean else 0
+            
+            # Extract educational comments
+            educational_comments = []
+            if has_unclean:
+                educational_comments = self._extract_educational_comments(unclean_data)
+            
+            # Validate samples
+            validation_errors = []
+            if has_clean:
+                _, clean_errors = self.validate_sample_data(clean_data, expression_name, "clean")
+                validation_errors.extend([f"Clean: {err}" for err in clean_errors])
+            
+            if has_unclean:
+                _, unclean_errors = self.validate_sample_data(unclean_data, expression_name, "unclean")
+                validation_errors.extend([f"Unclean: {err}" for err in unclean_errors])
+            
+            return SampleDataInfo(
+                expression_name=expression_name,
+                version=version or self.version_manager.default_version,
+                has_clean=has_clean,
+                has_unclean=has_unclean,
+                clean_rows=clean_rows,
+                unclean_rows=unclean_rows,
+                educational_comments=educational_comments,
+                validation_errors=validation_errors
+            )
+            
+        except Exception as e:
+            log_warning(f"[sample_data] Failed to get sample info for {expression_name}: {e}")
+            return SampleDataInfo(
+                expression_name=expression_name,
+                version=version or "unknown",
+                has_clean=False,
+                has_unclean=False,
+                clean_rows=0,
+                unclean_rows=0,
+                educational_comments=[],
+                validation_errors=[f"Failed to get sample info: {e}"]
+            )
+    
+    def create_sample_template(self, expression_name: str, columns: List[str]) -> Dict[str, Dict[str, Any]]:
+        """
+        Create a template for sample data
+        
+        Args:
+            expression_name: Name of the expression
+            columns: List of required columns
+            
+        Returns:
+            Dictionary with clean and unclean sample templates
+        """
+        try:
+            # Create clean sample template
+            clean_template = {}
+            for column in columns:
+                clean_template[column] = [f"sample_{column}_1", f"sample_{column}_2", f"sample_{column}_3"]
+            
+            # Create unclean sample template with educational comments
+            unclean_template = {}
+            for column in columns:
+                unclean_template[column] = [
+                    f"valid_{column}",
+                    None,  # Missing value
+                    f"invalid_{column}_type",
+                    f"extreme_{column}_value"
+                ]
+            
+            # Add educational comments
+            unclean_template["_comments"] = [
+                "# This is unclean sample data for testing error handling",
+                "# Row 1: Valid data",
+                "# Row 2: Missing values (None/null)",
+                "# Row 3: Invalid data types",
+                "# Row 4: Extreme or edge case values"
+            ]
+            
+            template = {
+                "clean": clean_template,
+                "unclean": unclean_template
+            }
+            
+            log_info(f"[sample_data] Created sample template for {expression_name}")
+            return template
+            
+        except Exception as e:
+            log_warning(f"[sample_data] Failed to create sample template for {expression_name}: {e}")
+            raise SampleDataError(f"Failed to create sample template: {e}")
+    
+    def _get_sample_data(self, expression_name: str, namespace: str, 
+                        version: Optional[str], sample_type: str) -> Optional[Dict[str, Any]]:
+        """Get raw sample data from expression file"""
+        try:
+            # Get expression file path
+            expression_path = self.namespace_manager.get_expression_file_path(
+                namespace, expression_name, version
+            )
+            
+            if not expression_path or not os.path.exists(expression_path):
+                return None
+            
+            # Validate integrity
+            if not self.integrity_manager.validate_integrity(expression_path):
+                log_warning(f"[sample_data] Integrity validation failed for {expression_path}")
+                return None
+            
+            # Parse expression file
+            with open(expression_path, 'r') as f:
+                content = yaml.safe_load(f)
+            
+            # Extract sample data
+            sample_section = content.get("sample", {})
+            return sample_section.get(sample_type)
+            
+        except Exception as e:
+            log_warning(f"[sample_data] Failed to get sample data from {expression_path}: {e}")
+            return None
+    
+    def _validate_clean_sample(self, df: pd.DataFrame, expression_name: str) -> List[str]:
+        """Validate clean sample data"""
+        errors = []
+        
+        # Check for missing values in clean sample
+        if df.isnull().any().any():
+            errors.append("Clean sample should not contain missing values")
+        
+        # Check for reasonable data types
+        for column in df.columns:
+            if column.startswith('_'):  # Skip metadata columns
+                continue
+                
+            series = df[column]
+            
+            # Check for mixed types (should be consistent in clean data)
+            unique_types = set(type(x).__name__ for x in series.dropna())
+            if len(unique_types) > 1:
+                errors.append(f"Column '{column}' has mixed data types in clean sample")
+        
+        return errors
+    
+    def _validate_unclean_sample(self, df: pd.DataFrame, expression_name: str) -> List[str]:
+        """Validate unclean sample data"""
+        errors = []
+        
+        # Unclean samples should have some issues for educational purposes
+        has_nulls = df.isnull().any().any()
+        has_mixed_types = False
+        
+        for column in df.columns:
+            if column.startswith('_'):  # Skip metadata columns
+                continue
+                
+            series = df[column]
+            unique_types = set(type(x).__name__ for x in series.dropna())
+            if len(unique_types) > 1:
+                has_mixed_types = True
+                break
+        
+        # Unclean samples should demonstrate common data issues
+        if not has_nulls and not has_mixed_types:
+            errors.append("Unclean sample should contain some data quality issues for educational purposes")
+        
+        return errors
+    
+    def _add_educational_comments(self, df: pd.DataFrame, expression_name: str) -> pd.DataFrame:
+        """Add educational comments to unclean sample data"""
+        try:
+            # Add a comments column with educational information
+            comments = []
+            
+            for i, row in df.iterrows():
+                comment_parts = []
+                
+                # Check for missing values
+                if row.isnull().any():
+                    comment_parts.append("Contains missing values")
+                
+                # Check for potential type issues
+                for col, val in row.items():
+                    if col.startswith('_'):
+                        continue
+                    if isinstance(val, str) and val.lower() in ['invalid', 'error', 'null']:
+                        comment_parts.append(f"'{col}' has invalid value")
+                
+                if not comment_parts:
+                    comment_parts.append("Valid data row")
+                
+                comments.append(" | ".join(comment_parts))
+            
+            # Add comments as a new column
+            df_with_comments = df.copy()
+            df_with_comments['_educational_comments'] = comments
+            
+            return df_with_comments
+            
+        except Exception as e:
+            log_warning(f"[sample_data] Failed to add educational comments: {e}")
+            return df
+    
+    def _extract_educational_comments(self, sample_data: Dict[str, Any]) -> List[str]:
+        """Extract educational comments from sample data"""
+        comments = []
+        
+        # Look for comment fields
+        if '_comments' in sample_data:
+            comments.extend(sample_data['_comments'])
+        
+        # Generate comments based on data patterns
+        try:
+            df = pd.DataFrame({k: v for k, v in sample_data.items() if not k.startswith('_')})
+            
+            if df.isnull().any().any():
+                comments.append("Contains missing values for null handling testing")
+            
+            for column in df.columns:
+                series = df[column]
+                unique_types = set(type(x).__name__ for x in series.dropna())
+                if len(unique_types) > 1:
+                    comments.append(f"Column '{column}' has mixed types for type validation testing")
+        
+        except Exception:
+            pass  # Ignore errors in comment extraction
+        
+        return comments
+    
+    def _generate_default_clean_sample(self, expression_name: str) -> pd.DataFrame:
+        """Generate default clean sample data when none exists"""
+        return pd.DataFrame({
+            "col_a": [1, 2, 3],
+            "col_b": [4, 5, 6],
+            "_info": [f"Default clean sample for '{expression_name}'"] * 3
+        })
+    
+    def _generate_default_unclean_sample(self, expression_name: str) -> pd.DataFrame:
+        """Generate default unclean sample data when none exists"""
+        return pd.DataFrame({
+            "col_a": [1, None, "invalid"],
+            "col_b": [4, 5, -999],
+            "_educational_comments": [
+                "Valid data row",
+                "Missing value in col_a",
+                "Invalid type in col_a, extreme value in col_b"
+            ],
+            "_info": [f"Default unclean sample for '{expression_name}'"] * 3
+        })
+
+
+# Global sample data manager instance
+_sample_data_manager = None
+
+def get_sample_data_manager() -> SampleDataManager:
+    """Get the global sample data manager instance"""
+    global _sample_data_manager
+    if _sample_data_manager is None:
+        _sample_data_manager = SampleDataManager()
+    return _sample_data_manager

additory/synthetic/__init__.py

@@ -0,0 +1,101 @@
+"""
+Additory Synthetic Data Generation Module
+
+This module provides polars-native synthetic data generation using regex patterns
+and distribution strategies. It supports hierarchical pattern resolution and
+industry-standard file formats (.properties and .toml).
+"""
+
+from .api import (
+    synth, 
+    config,
+    register_distribution_engine,
+    unregister_distribution_engine,
+    list_custom_distribution_engines
+)
+from .exceptions import (
+    SyntheticDataError,
+    PatternResolutionError,
+    ValidationError,
+    DistributionError,
+    FileFormatError,
+    PatternImportError,
+    SchemaParsingError
+)
+from .pattern_resolver import PatternHierarchyResolver, ResolutionTrace, PatternResolutionResult
+from .engines import (
+    DistributionEngine,
+    DistributionEngineFactory,
+    DistributionManager,
+    DistributionConfig,
+)
+from .generator import (
+    RegexGenerator,
+    PolarsGeneratorCore,
+    OutputConverter,
+    SyntheticDataGenerator,
+    GenerationConfig,
+)
+from .performance import (
+    PerformanceMonitor,
+    PerformanceOptimizer,
+    PerformanceMetrics,
+    PerformanceComparison,
+    performance_monitor,
+    performance_optimizer
+)
+from .polars_integration import (
+    PolarsIntegrationLayer,
+    optimize_conversion,
+    enhance_result,
+    optimize_context,
+    apply_expression,
+    optimize_memory,
+    validate_compatibility,
+    get_integration_stats,
+    cleanup_integration,
+    benchmark_integration
+)
+
+__all__ = [
+    'synth',
+    'config',
+    'register_distribution_engine',
+    'unregister_distribution_engine',
+    'list_custom_distribution_engines',
+    'SyntheticDataError',
+    'PatternResolutionError', 
+    'ValidationError',
+    'DistributionError',
+    'FileFormatError',
+    'PatternImportError',
+    'SchemaParsingError',
+    'PatternHierarchyResolver',
+    'ResolutionTrace',
+    'PatternResolutionResult',
+    'DistributionEngine',
+    'DistributionEngineFactory',
+    'DistributionManager',
+    'DistributionConfig',
+    'RegexGenerator',
+    'PolarsGeneratorCore',
+    'OutputConverter',
+    'SyntheticDataGenerator',
+    'GenerationConfig',
+    'PerformanceMonitor',
+    'PerformanceOptimizer',
+    'PerformanceMetrics',
+    'PerformanceComparison',
+    'performance_monitor',
+    'performance_optimizer',
+    'PolarsIntegrationLayer',
+    'optimize_conversion',
+    'enhance_result',
+    'optimize_context',
+    'apply_expression',
+    'optimize_memory',
+    'validate_compatibility',
+    'get_integration_stats',
+    'cleanup_integration',
+    'benchmark_integration'
+]