autoprepml 1.0.0__py3-none-any.whl

autoprepml/__init__.py

@@ -0,0 +1,33 @@
+"""AutoPrepML - Automated Data Preprocessing Pipeline
+
+A Python library for automatic detection, cleaning, and reporting of common
+data quality issues in machine learning pipelines.
+"""
+
+__version__ = "0.1.0"
+__author__ = "AutoPrepML Contributors"
+__license__ = "MIT"
+
+from .core import AutoPrepML
+from .text import TextPrepML
+from .timeseries import TimeSeriesPrepML
+from .graph import GraphPrepML
+from . import detection
+from . import cleaning
+from . import visualization
+from . import reports
+from . import config
+from . import llm_suggest
+
+__all__ = [
+    "AutoPrepML",
+    "TextPrepML",
+    "TimeSeriesPrepML",
+    "GraphPrepML",
+    "detection",
+    "cleaning",
+    "visualization",
+    "reports",
+    "config",
+    "llm_suggest",
+]

autoprepml/cleaning.py

@@ -0,0 +1,176 @@
+"""Cleaning and transformation functions for AutoPrepML"""
+import pandas as pd
+import numpy as np
+from sklearn.preprocessing import StandardScaler, MinMaxScaler, LabelEncoder
+
+
+def impute_missing(df: pd.DataFrame, strategy: str = 'auto', 
+                   numeric_strategy: str = 'median',
+                   categorical_strategy: str = 'mode') -> pd.DataFrame:
+    """Impute missing values in DataFrame.
+    
+    Args:
+        df: Input DataFrame
+        strategy: 'auto' (smart detection), 'median', 'mean', 'mode', 'drop'
+        numeric_strategy: Strategy for numeric columns when strategy='auto'
+        categorical_strategy: Strategy for categorical columns when strategy='auto'
+        
+    Returns:
+        DataFrame with imputed values
+    """
+    df_clean = df.copy()
+    
+    for col in df_clean.columns:
+        if df_clean[col].isnull().sum() == 0:
+            continue
+        
+        if strategy == 'drop':
+            df_clean = df_clean.dropna(subset=[col])
+            continue
+        
+        # Auto-detect strategy based on dtype
+        if strategy == 'auto':
+            if pd.api.types.is_numeric_dtype(df_clean[col]):
+                current_strategy = numeric_strategy
+            else:
+                current_strategy = categorical_strategy
+        else:
+            current_strategy = strategy
+        
+        # Apply imputation
+        if current_strategy == 'median' and pd.api.types.is_numeric_dtype(df_clean[col]):
+            fill_value = df_clean[col].median()
+        elif current_strategy == 'mean' and pd.api.types.is_numeric_dtype(df_clean[col]):
+            fill_value = df_clean[col].mean()
+        elif current_strategy == 'mode':
+            mode_values = df_clean[col].mode()
+            fill_value = mode_values.iloc[0] if len(mode_values) > 0 else ''
+        else:
+            fill_value = 0 if pd.api.types.is_numeric_dtype(df_clean[col]) else ''
+        
+        df_clean[col] = df_clean[col].fillna(fill_value)
+    
+    return df_clean
+
+
+def scale_features(df: pd.DataFrame, method: str = 'standard', 
+                   exclude_cols: list = None) -> pd.DataFrame:
+    """Scale numeric features.
+    
+    Args:
+        df: Input DataFrame
+        method: 'standard' (z-score normalization) or 'minmax' (0-1 scaling)
+        exclude_cols: List of column names to exclude from scaling
+        
+    Returns:
+        DataFrame with scaled numeric features
+    """
+    df_scaled = df.copy()
+    exclude_cols = exclude_cols or []
+    
+    numeric_cols = df_scaled.select_dtypes(include=[np.number]).columns.tolist()
+    cols_to_scale = [col for col in numeric_cols if col not in exclude_cols]
+    
+    if not cols_to_scale:
+        return df_scaled
+    
+    if method == 'standard':
+        scaler = StandardScaler()
+    elif method == 'minmax':
+        scaler = MinMaxScaler()
+    else:
+        raise ValueError(f"Unknown scaling method: {method}")
+    
+    df_scaled[cols_to_scale] = scaler.fit_transform(df_scaled[cols_to_scale])
+    
+    return df_scaled
+
+
+def encode_categorical(df: pd.DataFrame, method: str = 'label', 
+                      exclude_cols: list = None) -> pd.DataFrame:
+    """Encode categorical features.
+    
+    Args:
+        df: Input DataFrame
+        method: 'label' (label encoding) or 'onehot' (one-hot encoding)
+        exclude_cols: List of column names to exclude from encoding
+        
+    Returns:
+        DataFrame with encoded categorical features
+    """
+    df_encoded = df.copy()
+    exclude_cols = exclude_cols or []
+    
+    categorical_cols = df_encoded.select_dtypes(include=['object', 'category']).columns.tolist()
+    cols_to_encode = [col for col in categorical_cols if col not in exclude_cols]
+    
+    if not cols_to_encode:
+        return df_encoded
+    
+    if method == 'label':
+        for col in cols_to_encode:
+            le = LabelEncoder()
+            df_encoded[col] = le.fit_transform(df_encoded[col].astype(str))
+    
+    elif method == 'onehot':
+        df_encoded = pd.get_dummies(df_encoded, columns=cols_to_encode, drop_first=True)
+    
+    else:
+        raise ValueError(f"Unknown encoding method: {method}")
+    
+    return df_encoded
+
+
+def balance_classes(df: pd.DataFrame, target_col: str, method: str = 'oversample') -> pd.DataFrame:
+    """Balance class distribution in target column.
+    
+    Args:
+        df: Input DataFrame
+        target_col: Name of target column
+        method: 'oversample' (duplicate minority), 'undersample' (reduce majority)
+        
+    Returns:
+        DataFrame with balanced classes
+    """
+    if target_col not in df.columns:
+        raise ValueError(f"Target column '{target_col}' not found in DataFrame")
+    
+    df_balanced = df.copy()
+    value_counts = df_balanced[target_col].value_counts()
+    
+    if method == 'oversample':
+        max_count = value_counts.max()
+        dfs = []
+        for class_value in value_counts.index:
+            class_df = df_balanced[df_balanced[target_col] == class_value]
+            count_diff = max_count - len(class_df)
+            if count_diff > 0:
+                oversampled = class_df.sample(n=count_diff, replace=True, random_state=42)
+                dfs.extend([class_df, oversampled])
+            else:
+                dfs.append(class_df)
+        df_balanced = pd.concat(dfs, ignore_index=True)
+    elif method == 'undersample':
+        min_count = value_counts.min()
+        dfs = [
+            df_balanced[df_balanced[target_col] == class_value].sample(n=min_count, random_state=42)
+            for class_value in value_counts.index
+        ]
+        df_balanced = pd.concat(dfs, ignore_index=True)
+    else:
+        raise ValueError(f"Unknown balancing method: {method}")
+    
+    return df_balanced.sample(frac=1, random_state=42).reset_index(drop=True)
+
+
+def remove_outliers(df: pd.DataFrame, outlier_indices: list) -> pd.DataFrame:
+    """Remove rows identified as outliers.
+    
+    Args:
+        df: Input DataFrame
+        outlier_indices: List of row indices to remove
+        
+    Returns:
+        DataFrame with outliers removed
+    """
+    return df.drop(index=outlier_indices).reset_index(drop=True)

autoprepml/cli.py

@@ -0,0 +1,166 @@
+"""Command-line interface for AutoPrepML"""
+import argparse
+import sys
+import pandas as pd
+from pathlib import Path
+from .core import AutoPrepML
+
+
+def main():
+    """Main CLI entrypoint."""
+    parser = argparse.ArgumentParser(
+        description='AutoPrepML - Automated Data Preprocessing Pipeline',
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Basic cleaning
+  autoprepml --input data.csv --output cleaned.csv
+  
+  # Classification task with target column
+  autoprepml --input train.csv --output clean_train.csv --task classification --target label
+  
+  # Generate HTML report
+  autoprepml --input data.csv --output cleaned.csv --report report.html
+  
+  # Use custom config
+  autoprepml --input data.csv --output cleaned.csv --config config.yaml
+        """
+    )
+    
+    parser.add_argument(
+        '--input', '-i',
+        required=True,
+        help='Input CSV file path'
+    )
+    
+    parser.add_argument(
+        '--output', '-o',
+        required=True,
+        help='Output cleaned CSV file path'
+    )
+    
+    parser.add_argument(
+        '--report', '-r',
+        help='Output report file path (.html or .json)'
+    )
+    
+    parser.add_argument(
+        '--task',
+        choices=['classification', 'regression'],
+        help='ML task type (affects preprocessing strategy)'
+    )
+    
+    parser.add_argument(
+        '--target',
+        help='Name of target column (for classification/regression tasks)'
+    )
+    
+    parser.add_argument(
+        '--config', '-c',
+        help='Path to YAML/JSON configuration file'
+    )
+    
+    parser.add_argument(
+        '--no-plots',
+        action='store_true',
+        help='Disable plot generation in reports'
+    )
+    
+    parser.add_argument(
+        '--detect-only',
+        action='store_true',
+        help='Only run detection, do not clean data'
+    )
+    
+    parser.add_argument(
+        '--verbose', '-v',
+        action='store_true',
+        help='Enable verbose logging'
+    )
+    
+    args = parser.parse_args()
+    
+    # Validate inputs
+    input_path = Path(args.input)
+    if not input_path.exists():
+        print(f"❌ Error: Input file not found: {args.input}", file=sys.stderr)
+        sys.exit(1)
+    
+    if input_path.suffix.lower() != '.csv':
+        print("❌ Error: Input file must be a CSV file", file=sys.stderr)
+        sys.exit(1)
+    
+    # Validate report format
+    if args.report:
+        report_path = Path(args.report)
+        if report_path.suffix.lower() not in ['.html', '.json']:
+            print("❌ Error: Report file must be .html or .json", file=sys.stderr)
+            sys.exit(1)
+    
+    # Load data
+    try:
+        print(f"📂 Loading data from {args.input}...")
+        df = pd.read_csv(args.input)
+        print(f"✅ Loaded {len(df)} rows, {len(df.columns)} columns")
+    except Exception as e:
+        print(f"❌ Error loading CSV: {e}", file=sys.stderr)
+        sys.exit(1)
+    
+    # Initialize AutoPrepML
+    try:
+        prep = AutoPrepML(df, config_path=args.config)
+        
+        if args.no_plots:
+            prep.config['reporting']['include_plots'] = False
+        
+        # Detection phase
+        print("\n🔍 Running detection...")
+        detection_results = prep.detect(target_col=args.target)
+        
+        # Print detection summary
+        missing_count = len(detection_results.get('missing_values', {}))
+        outlier_count = detection_results.get('outliers', {}).get('outlier_count', 0)
+        
+        print(f"   • Missing values: {missing_count} columns affected")
+        print(f"   • Outliers detected: {outlier_count} rows")
+        
+        if args.target and 'class_imbalance' in detection_results:
+            imbalance = detection_results['class_imbalance']
+            status = "⚠ Imbalanced" if imbalance['is_imbalanced'] else "✓ Balanced"
+            print(f"   • Class distribution: {status}")
+        
+        if args.detect_only:
+            print("\n✅ Detection complete (--detect-only mode)")
+            if args.report:
+                prep.save_report(args.report)
+                print(f"📄 Report saved to {args.report}")
+            sys.exit(0)
+        
+        # Cleaning phase
+        print("\n🧹 Cleaning data...")
+        clean_df, report = prep.clean(task=args.task, target_col=args.target)
+        
+        # Save cleaned data
+        output_path = Path(args.output)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        clean_df.to_csv(args.output, index=False)
+        print(f"✅ Cleaned data saved to {args.output}")
+        print(f"   Shape: {clean_df.shape[0]} rows × {clean_df.shape[1]} columns")
+        
+        # Save report
+        if args.report:
+            prep.save_report(args.report)
+            print(f"📄 Report saved to {args.report}")
+        
+        print("\n🎉 AutoPrepML completed successfully!")
+        
+    except Exception as e:
+        print(f"❌ Error during preprocessing: {e}", file=sys.stderr)
+        if args.verbose:
+            import traceback
+            traceback.print_exc()
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

autoprepml/config.py

@@ -0,0 +1,95 @@
+"""Configuration management for AutoPrepML"""
+import os
+import json
+import yaml
+from typing import Dict, Any, Optional
+
+
+DEFAULT_CONFIG = {
+    'cleaning': {
+        'missing_strategy': 'auto',
+        'numeric_strategy': 'median',
+        'categorical_strategy': 'mode',
+        'outlier_method': 'iforest',
+        'outlier_contamination': 0.05,
+        'remove_outliers': False,
+        'scale_method': 'standard',
+        'encode_method': 'label',
+        'balance_method': 'oversample'
+    },
+    'detection': {
+        'outlier_method': 'iforest',
+        'contamination': 0.05,
+        'zscore_threshold': 3.0,
+        'imbalance_threshold': 0.3
+    },
+    'reporting': {
+        'format': 'html',
+        'include_plots': True,
+        'plot_style': 'seaborn',
+        'output_dir': './reports'
+    },
+    'logging': {
+        'enabled': True,
+        'level': 'INFO',
+        'file': 'autoprepml.log'
+    }
+}
+
+
+def load_config(config_path: Optional[str] = None) -> Dict[str, Any]:
+    """Load configuration from YAML or JSON file.
+    
+    Args:
+        config_path: Path to config file. If None, returns default config.
+        
+    Returns:
+        Configuration dictionary
+    """
+    if config_path is None:
+        return DEFAULT_CONFIG.copy()
+    
+    if not os.path.exists(config_path):
+        raise FileNotFoundError(f"Config file not found: {config_path}")
+    
+    with open(config_path, 'r', encoding='utf-8') as f:
+        if config_path.endswith('.yaml') or config_path.endswith('.yml'):
+            user_config = yaml.safe_load(f)
+        elif config_path.endswith('.json'):
+            user_config = json.load(f)
+        else:
+            raise ValueError("Config file must be YAML or JSON")
+    
+    # Merge with defaults (user config overrides defaults)
+    config = DEFAULT_CONFIG.copy()
+    if user_config:
+        for section, values in user_config.items():
+            if section in config and isinstance(config[section], dict):
+                config[section].update(values)
+            else:
+                config[section] = values
+    
+    return config
+
+
+def save_config(config: Dict[str, Any], output_path: str) -> None:
+    """Save configuration to YAML file.
+    
+    Args:
+        config: Configuration dictionary
+        output_path: Path to save config file
+    """
+    with open(output_path, 'w', encoding='utf-8') as f:
+        if output_path.endswith('.json'):
+            json.dump(config, f, indent=2)
+        else:
+            yaml.dump(config, f, default_flow_style=False, sort_keys=False)
+
+
+def get_default_config() -> Dict[str, Any]:
+    """Return a copy of the default configuration.
+    
+    Returns:
+        Default configuration dictionary
+    """
+    return DEFAULT_CONFIG.copy()

autoprepml/core.py

@@ -0,0 +1,194 @@
+"""Core high-level interface for AutoPrepML"""
+from typing import Optional, Dict, Any, Tuple
+import pandas as pd
+import logging
+from datetime import datetime
+
+from . import detection
+from . import cleaning
+from . import visualization
+from .config import load_config, DEFAULT_CONFIG
+
+
+# Setup logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger('autoprepml')
+
+
+class AutoPrepML:
+    """Main AutoPrepML class for automated data preprocessing.
+    
+    Example:
+        >>> df = pd.read_csv('data.csv')
+        >>> prep = AutoPrepML(df)
+        >>> clean_df, report = prep.clean(task='classification', target_col='label')
+        >>> prep.save_report('report.html')
+    """
+    
+    def __init__(self, df: pd.DataFrame, config: Optional[Dict[str, Any]] = None, 
+                 config_path: Optional[str] = None):
+        """Initialize AutoPrepML with a DataFrame.
+        
+        Args:
+            df: Input DataFrame to preprocess
+            config: Optional configuration dictionary
+            config_path: Optional path to YAML/JSON config file
+        """
+        self.original_df = df.copy()
+        self.df = df.copy()
+        self.cleaned_df = None
+        self.log = []
+        self.detection_results = {}
+        self.plots = {}
+        
+        # Load configuration
+        if config_path:
+            self.config = load_config(config_path)
+        elif config:
+            self.config = config
+        else:
+            self.config = DEFAULT_CONFIG.copy()
+        
+        self._log_action('initialized', {'shape': df.shape, 'columns': list(df.columns)})
+    
+    def _log_action(self, action: str, details: Any) -> None:
+        """Internal logging helper."""
+        entry = {
+            'timestamp': datetime.now().isoformat(),
+            'action': action,
+            'details': details
+        }
+        self.log.append(entry)
+        logger.info(f"{action}: {details}")
+    
+    def detect(self, target_col: Optional[str] = None) -> Dict[str, Any]:
+        """Run all detection functions.
+        
+        Args:
+            target_col: Optional target column for imbalance detection
+            
+        Returns:
+            Dictionary containing all detection results
+        """
+        self.detection_results = detection.detect_all(self.df, target_col)
+        self._log_action('detection_complete', self.detection_results)
+        return self.detection_results
+    
+    def clean(self, task: Optional[str] = None, target_col: Optional[str] = None,
+              auto: bool = True) -> Tuple[pd.DataFrame, Dict[str, Any]]:
+        """Clean the dataset automatically.
+        
+        Args:
+            task: 'classification', 'regression', or None
+            target_col: Name of target column (for imbalance handling)
+            auto: If True, apply all cleaning steps automatically
+            
+        Returns:
+            Tuple of (cleaned_df, report_dict)
+        """
+        df_clean = self.df.copy()
+        
+        # Run detection first
+        if not self.detection_results:
+            self.detect(target_col)
+        
+        # Step 1: Handle missing values
+        if self.detection_results.get('missing_values'):
+            strategy = self.config['cleaning']['missing_strategy']
+            df_clean = cleaning.impute_missing(df_clean, strategy=strategy)
+            self._log_action('imputed_missing', {'strategy': strategy})
+        
+        # Step 2: Handle outliers (optional)
+        outliers = self.detection_results.get('outliers', {})
+        if outliers.get('outlier_count', 0) > 0 and self.config['cleaning']['remove_outliers']:
+            df_clean = cleaning.remove_outliers(df_clean, outliers['outlier_indices'])
+            self._log_action('removed_outliers', {'count': outliers['outlier_count']})
+        
+        # Step 3: Encode categorical variables
+        df_clean = cleaning.encode_categorical(df_clean, 
+                                               method=self.config['cleaning']['encode_method'],
+                                               exclude_cols=[target_col] if target_col else None)
+        self._log_action('encoded_categorical', {'method': self.config['cleaning']['encode_method']})
+        
+        # Step 4: Scale features
+        df_clean = cleaning.scale_features(df_clean, 
+                                           method=self.config['cleaning']['scale_method'],
+                                           exclude_cols=[target_col] if target_col else None)
+        self._log_action('scaled_features', {'method': self.config['cleaning']['scale_method']})
+        
+        # Step 5: Balance classes (if classification task and imbalanced)
+        if task == 'classification' and target_col:
+            imbalance = self.detection_results.get('class_imbalance', {})
+            if imbalance.get('is_imbalanced'):
+                df_clean = cleaning.balance_classes(df_clean, target_col,
+                                                    method=self.config['cleaning']['balance_method'])
+                self._log_action('balanced_classes', {'method': self.config['cleaning']['balance_method']})
+        
+        self.cleaned_df = df_clean
+        
+        # Generate report
+        report = self.report()
+        
+        return df_clean, report
+    
+    def summary(self) -> Dict[str, Any]:
+        """Get a quick summary of the dataset.
+        
+        Returns:
+            Dictionary with basic dataset statistics
+        """
+        return {
+            'shape': self.df.shape,
+            'columns': list(self.df.columns),
+            'dtypes': self.df.dtypes.astype(str).to_dict(),
+            'missing_values': detection.detect_missing(self.df),
+            'numeric_columns': self.df.select_dtypes(include=['number']).columns.tolist(),
+            'categorical_columns': self.df.select_dtypes(include=['object', 'category']).columns.tolist()
+        }
+    
+    def report(self, include_plots: bool = True) -> Dict[str, Any]:
+        """Generate comprehensive preprocessing report.
+        
+        Args:
+            include_plots: Whether to generate visualization plots
+            
+        Returns:
+            Complete report dictionary
+        """
+        report = {
+            'timestamp': datetime.now().isoformat(),
+            'original_shape': self.original_df.shape,
+            'cleaned_shape': self.cleaned_df.shape if self.cleaned_df is not None else None,
+            'detection_results': self.detection_results,
+            'logs': self.log,
+            'config': self.config
+        }
+        
+        # Generate plots if requested
+        if include_plots and self.config['reporting']['include_plots']:
+            outlier_indices = self.detection_results.get('outliers', {}).get('outlier_indices', [])
+            self.plots = visualization.generate_all_plots(self.original_df, outlier_indices)
+            report['plots'] = self.plots
+        
+        return report
+    
+    def save_report(self, output_path: str) -> None:
+        """Save report to file.
+        
+        Args:
+            output_path: Path to save report (supports .json, .html)
+        """
+        from .reports import generate_json_report, generate_html_report
+        
+        report = self.report(include_plots=True)
+        
+        if output_path.endswith('.json'):
+            with open(output_path, 'w', encoding='utf-8') as f:
+                f.write(generate_json_report(report))
+        elif output_path.endswith('.html'):
+            with open(output_path, 'w', encoding='utf-8') as f:
+                f.write(generate_html_report(report))
+        else:
+            raise ValueError("Output path must end with .json or .html")
+        
+        self._log_action('saved_report', {'path': output_path})