additory 0.1.0a3__py3-none-any.whl → 0.1.1a1__py3-none-any.whl

additory/analysis/scan.py

@@ -1,400 +0,0 @@
-"""
-Data Profiling and Analysis
-
-Provides comprehensive data profiling through the scan() function.
-"""
-
-from dataclasses import dataclass, field
-from typing import Dict, List, Optional, Union, Any
-import json
-import polars as pl
-import pandas as pd
-import numpy as np
-
-from additory.common.backend import detect_backend, to_polars
-from additory.analysis.distributions import (
-    detect_distributions,
-    DistributionFit
-)
-from additory.analysis.correlations import (
-    calculate_correlations,
-    CorrelationResult
-)
-from additory.analysis.cardinality import (
-    analyze_all_cardinality,
-    CardinalityInfo
-)
-from additory.analysis.quality import (
-    analyze_all_quality,
-    QualityMetrics,
-    is_numeric_dtype
-)
-
-# Scan presets for common use cases
-SCAN_PRESETS = {
-    'quick': {
-        'detect_distributions_flag': False,
-        'detect_correlations_flag': False,
-        'detect_cardinality_flag': True,
-        'description': 'Quick scan: quality + cardinality only'
-    },
-    'distributions': {
-        'detect_distributions_flag': True,
-        'detect_correlations_flag': False,
-        'detect_cardinality_flag': False,
-        'description': 'Distribution-focused: detect distributions only'
-    },
-    'correlations': {
-        'detect_distributions_flag': False,
-        'detect_correlations_flag': True,
-        'detect_cardinality_flag': False,
-        'description': 'Correlation-focused: correlation analysis only'
-    },
-    'full': {
-        'detect_distributions_flag': True,
-        'detect_correlations_flag': True,
-        'detect_cardinality_flag': True,
-        'description': 'Full analysis: all features enabled'
-    },
-    'minimal': {
-        'detect_distributions_flag': False,
-        'detect_correlations_flag': False,
-        'detect_cardinality_flag': False,
-        'description': 'Minimal scan: quality metrics only'
-    }
-}
-
-
-@dataclass
-class ColumnInfo:
-    """Information about a single column"""
-    name: str
-    dtype: str
-    null_count: int
-    null_percentage: float
-    unique_count: int
-    
-    def __repr__(self):
-        return f"ColumnInfo(name='{self.name}', dtype='{self.dtype}', nulls={self.null_percentage:.1f}%)"
-
-
-@dataclass
-class ScanResult:
-    """
-    Comprehensive scan results for a DataFrame
-    
-    Contains all analysis results including distributions, correlations,
-    cardinality, and quality metrics.
-    """
-    # Basic info
-    shape: tuple
-    columns: List[ColumnInfo]
-    
-    # Analysis results
-    distributions: Dict[str, List[DistributionFit]] = field(default_factory=dict)
-    correlations: List[CorrelationResult] = field(default_factory=list)
-    cardinality: Dict[str, CardinalityInfo] = field(default_factory=dict)
-    quality: Dict[str, QualityMetrics] = field(default_factory=dict)
-    
-    # Metadata
-    preset_used: Optional[str] = None
-    analysis_enabled: Dict[str, bool] = field(default_factory=dict)
-    
-    def summary(self) -> str:
-        """Generate a human-readable summary of the scan results"""
-        lines = []
-        lines.append(f"DataFrame Scan Results")
-        lines.append(f"Shape: {self.shape[0]:,} rows × {self.shape[1]} columns")
-        lines.append("")
-        
-        if self.preset_used:
-            preset_desc = SCAN_PRESETS.get(self.preset_used, {}).get('description', 'Custom preset')
-            lines.append(f"Preset: {self.preset_used} ({preset_desc})")
-            lines.append("")
-        
-        # Column overview
-        lines.append("Columns:")
-        for col in self.columns:
-            lines.append(f"  {col.name}: {col.dtype} ({col.null_percentage:.1f}% null, {col.unique_count:,} unique)")
-        lines.append("")
-        
-        # Distributions
-        if self.distributions:
-            lines.append("Top Distributions:")
-            for col_name, fits in self.distributions.items():
-                if fits:
-                    best_fit = fits[0]
-                    lines.append(f"  {col_name}: {best_fit.distribution} (score: {best_fit.score:.3f})")
-            lines.append("")
-        
-        # Correlations
-        if self.correlations:
-            lines.append("Strong Correlations:")
-            strong_corrs = [c for c in self.correlations if abs(c.correlation) >= 0.7]
-            for corr in strong_corrs[:5]:  # Top 5
-                lines.append(f"  {corr.column1} ↔ {corr.column2}: {corr.correlation:.3f} ({corr.method})")
-            if len(strong_corrs) > 5:
-                lines.append(f"  ... and {len(strong_corrs) - 5} more")
-            lines.append("")
-        
-        # Cardinality insights
-        if self.cardinality:
-            high_card = [name for name, info in self.cardinality.items() 
-                        if info.classification == 'high']
-            if high_card:
-                lines.append(f"High Cardinality Columns: {', '.join(high_card)}")
-                lines.append("")
-        
-        return "\n".join(lines)
-    
-    def to_dict(self) -> Dict[str, Any]:
-        """Convert scan results to dictionary format"""
-        return {
-            'shape': self.shape,
-            'columns': [
-                {
-                    'name': col.name,
-                    'dtype': col.dtype,
-                    'null_count': col.null_count,
-                    'null_percentage': col.null_percentage,
-                    'unique_count': col.unique_count
-                }
-                for col in self.columns
-            ],
-            'distributions': {
-                col_name: [
-                    {
-                        'distribution': fit.distribution,
-                        'score': fit.score,
-                        'parameters': fit.parameters
-                    }
-                    for fit in fits
-                ]
-                for col_name, fits in self.distributions.items()
-            },
-            'correlations': [
-                {
-                    'column1': corr.column1,
-                    'column2': corr.column2,
-                    'correlation': corr.correlation,
-                    'method': corr.method,
-                    'p_value': corr.p_value
-                }
-                for corr in self.correlations
-            ],
-            'cardinality': {
-                col_name: {
-                    'unique_count': info.unique_count,
-                    'total_count': info.total_count,
-                    'unique_ratio': info.unique_ratio,
-                    'classification': info.classification,
-                    'top_values': info.top_values
-                }
-                for col_name, info in self.cardinality.items()
-            },
-            'quality': {
-                col_name: {
-                    'missing_count': metrics.missing_count,
-                    'missing_percentage': metrics.missing_percentage,
-                    'data_type': metrics.data_type,
-                    'summary_stats': metrics.summary_stats
-                }
-                for col_name, metrics in self.quality.items()
-            },
-            'metadata': {
-                'preset_used': self.preset_used,
-                'analysis_enabled': self.analysis_enabled
-            }
-        }
-    
-    def to_json(self, indent: int = 2) -> str:
-        """Convert scan results to JSON string"""
-        return json.dumps(self.to_dict(), indent=indent, default=str)
-
-
-def scan(
-    df: Union[pl.DataFrame, pd.DataFrame, Any],
-    preset: Optional[str] = None,
-    detect_distributions_flag: bool = True,
-    detect_correlations_flag: bool = True,
-    detect_cardinality_flag: bool = True,
-    top_n_distributions: int = 3,
-    correlation_methods: List[str] = None,
-    correlation_threshold: float = 0.3,
-    cardinality_top_n: int = 10,
-    verbose: bool = True
-) -> ScanResult:
-    """
-    Scan a DataFrame to detect distributions, correlations, and cardinality.
-    
-    Accepts pandas, polars, or cuDF DataFrames. Automatically converts to Polars
-    for processing. Returns ScanResult with analysis results.
-    
-    This function provides comprehensive data profiling including:
-    - Distribution detection for numeric columns
-    - Correlation analysis between columns
-    - Cardinality analysis (unique values)
-    - Data quality metrics
-    
-    Args:
-        df: DataFrame to analyze (pandas, polars, or cuDF)
-        preset: Optional preset ('quick', 'distributions', 'correlations', 'full', 'minimal')
-        detect_distributions_flag: Whether to detect distributions (default: True)
-        detect_correlations_flag: Whether to calculate correlations (default: True)
-        detect_cardinality_flag: Whether to analyze cardinality (default: True)
-        top_n_distributions: Number of top distributions to return per column (default: 3)
-        correlation_methods: Correlation methods to use (default: ['pearson', 'spearman'])
-        correlation_threshold: Minimum correlation to report (default: 0.3)
-        cardinality_top_n: Number of top values to return per column (default: 10)
-        verbose: Whether to print progress messages (default: True)
-        
-    Returns:
-        ScanResult object containing all analysis results
-        
-    Presets:
-        - 'quick': Quality + cardinality only (fast)
-        - 'distributions': Distribution detection only
-        - 'correlations': Correlation analysis only
-        - 'full': All analyses enabled
-        - 'minimal': Quality metrics only (fastest)
-        
-    Example:
-        >>> import pandas as pd
-        >>> from additory.analysis.scan import scan
-        >>> 
-        >>> # Works with pandas
-        >>> df = pd.DataFrame({
-        ...     'age': [25, 30, 35, 40, 45],
-        ...     'income': [50000, 60000, 70000, 80000, 90000],
-        ...     'category': ['A', 'B', 'A', 'B', 'A']
-        ... })
-        >>> 
-        >>> result = scan(df)
-        >>> print(result.summary())
-        >>> 
-        >>> # Use presets
-        >>> result = scan(df, preset='quick')
-        >>> result = scan(df, preset='distributions', top_n_distributions=5)
-    """
-    # Handle preset configuration
-    if preset:
-        if preset not in SCAN_PRESETS:
-            available = ', '.join(SCAN_PRESETS.keys())
-            raise ValueError(f"Unknown preset '{preset}'. Available presets: {available}")
-        
-        preset_config = SCAN_PRESETS[preset]
-        # Override flags with preset values (but allow explicit overrides)
-        if 'detect_distributions_flag' not in locals() or detect_distributions_flag is True:
-            detect_distributions_flag = preset_config['detect_distributions_flag']
-        if 'detect_correlations_flag' not in locals() or detect_correlations_flag is True:
-            detect_correlations_flag = preset_config['detect_correlations_flag']
-        if 'detect_cardinality_flag' not in locals() or detect_cardinality_flag is True:
-            detect_cardinality_flag = preset_config['detect_cardinality_flag']
-    
-    # Set default correlation methods
-    if correlation_methods is None:
-        correlation_methods = ['pearson', 'spearman']
-    
-    # Convert to Polars for processing
-    original_backend = detect_backend(df)
-    if verbose:
-        print(f"Scanning {original_backend} DataFrame with shape {df.shape}")
-    
-    df_polars = to_polars(df)
-    
-    # Memory cleanup: delete original if converted
-    if original_backend != 'polars':
-        del df
-        import gc
-        gc.collect()
-    
-    # Get basic info
-    shape = df_polars.shape
-    column_names = df_polars.columns
-    
-    # Analyze column info
-    columns = []
-    for col_name in column_names:
-        col_series = df_polars[col_name]
-        dtype = str(col_series.dtype)
-        null_count = col_series.null_count()
-        null_percentage = (null_count / shape[0]) * 100
-        unique_count = col_series.n_unique()
-        
-        columns.append(ColumnInfo(
-            name=col_name,
-            dtype=dtype,
-            null_count=null_count,
-            null_percentage=null_percentage,
-            unique_count=unique_count
-        ))
-    
-    # Initialize result
-    result = ScanResult(
-        shape=shape,
-        columns=columns,
-        preset_used=preset,
-        analysis_enabled={
-            'distributions': detect_distributions_flag,
-            'correlations': detect_correlations_flag,
-            'cardinality': detect_cardinality_flag
-        }
-    )
-    
-    # Quality analysis (always performed)
-    if verbose:
-        print("Analyzing data quality...")
-    result.quality = analyze_all_quality(df_polars)
-    
-    # Distribution analysis
-    if detect_distributions_flag:
-        if verbose:
-            print("Detecting distributions...")
-        
-        # Get numeric columns
-        numeric_columns = [col.name for col in columns 
-                          if is_numeric_dtype(df_polars[col.name].dtype)]
-        
-        if numeric_columns:
-            result.distributions = detect_distributions(
-                df_polars, 
-                columns=numeric_columns,
-                top_n=top_n_distributions
-            )
-        elif verbose:
-            print("No numeric columns found for distribution analysis")
-    
-    # Correlation analysis
-    if detect_correlations_flag:
-        if verbose:
-            print("Calculating correlations...")
-        
-        # Get numeric columns for correlation
-        numeric_columns = [col.name for col in columns 
-                          if is_numeric_dtype(df_polars[col.name].dtype)]
-        
-        if len(numeric_columns) >= 2:
-            result.correlations = calculate_correlations(
-                df_polars,
-                columns=numeric_columns,
-                methods=correlation_methods,
-                threshold=correlation_threshold
-            )
-        elif verbose:
-            print(f"Need at least 2 numeric columns for correlation analysis (found {len(numeric_columns)})")
-    
-    # Cardinality analysis
-    if detect_cardinality_flag:
-        if verbose:
-            print("Analyzing cardinality...")
-        result.cardinality = analyze_all_cardinality(df_polars, top_n=cardinality_top_n)
-    
-    if verbose:
-        print("Scan complete!")
-    
-    # Final memory cleanup
-    del df_polars
-    import gc
-    gc.collect()
-    
-    return result