cleanflow-kit 1.1.0__py3-none-any.whl

dataclean/data_loader.py

@@ -0,0 +1,194 @@
+"""
+Data Loader Module
+==================
+Handles loading and initial validation of datasets.
+"""
+
+import logging
+logger = logging.getLogger(__name__)
+import pandas as pd
+import numpy as np
+from pathlib import Path
+from typing import Union, Optional, Dict, Any
+from ._compat import normalize_string_columns
+
+
+class DataLoader:
+    """Load and validate datasets from various file formats."""
+    
+    def __init__(self):
+        self.df: Optional[pd.DataFrame] = None
+        self.file_path: Optional[str] = None
+        self.validation_report: Dict[str, Any] = {}
+    
+    def load(
+        self, 
+        source: Union[str, pd.DataFrame], 
+        **kwargs
+    ) -> pd.DataFrame:
+        """
+        Load dataset from file path or DataFrame.
+        
+        Parameters:
+        -----------
+        source : str or pd.DataFrame
+            File path (CSV/Excel) or existing DataFrame
+        **kwargs : dict
+            Additional arguments passed to pandas read functions
+            
+        Returns:
+        --------
+        pd.DataFrame : Loaded dataset
+        """
+        if isinstance(source, pd.DataFrame):
+            self.df = source.copy()
+            self.file_path = "DataFrame input"
+        elif isinstance(source, str):
+            self.file_path = source
+            self.df = self._load_from_file(source, **kwargs)
+        else:
+            raise ValueError(f"Unsupported source type: {type(source)}")
+        
+        return normalize_string_columns(self.df)
+    
+    def _load_from_file(self, file_path: str, **kwargs) -> pd.DataFrame:
+        """Load data from file based on extension."""
+        path = Path(file_path)
+        
+        if not path.exists():
+            raise FileNotFoundError(f"File not found: {file_path}")
+        
+        extension = path.suffix.lower()
+        
+        if extension == '.csv':
+            return pd.read_csv(file_path, **kwargs)
+        elif extension in ['.xlsx', '.xls']:
+            return pd.read_excel(file_path, **kwargs)
+        elif extension == '.json':
+            return pd.read_json(file_path, **kwargs)
+        elif extension == '.parquet':
+            return pd.read_parquet(file_path, **kwargs)
+        else:
+            raise ValueError(f"Unsupported file format: {extension}")
+    
+    def validate(self) -> Dict[str, Any]:
+        """
+        Perform initial validation on loaded dataset.
+        
+        Returns:
+        --------
+        dict : Validation report with dataset information
+        """
+        if self.df is None:
+            raise ValueError("No dataset loaded. Call load() first.")
+        
+        df = self.df
+        
+        # Basic info
+        self.validation_report = {
+            "shape": df.shape,
+            "columns": list(df.columns),
+            "dtypes": df.dtypes.to_dict(),
+            "memory_usage_mb": df.memory_usage(deep=True).sum() / (1024 * 1024),
+        }
+        
+        # Missing values
+        missing = df.isnull().sum()
+        missing_pct = (missing / len(df) * 100).round(2)
+        self.validation_report["missing_values"] = {
+            "counts": missing[missing > 0].to_dict(),
+            "percentages": missing_pct[missing_pct > 0].to_dict(),
+            "total_missing_cells": int(missing.sum()),
+            "columns_with_missing": int((missing > 0).sum())
+        }
+        
+        # Duplicates
+        duplicate_count = df.duplicated().sum()
+        self.validation_report["duplicates"] = {
+            "count": int(duplicate_count),
+            "percentage": round(duplicate_count / len(df) * 100, 2)
+        }
+        
+        # Data type analysis
+        numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+        categorical_cols = df.select_dtypes(include=['object', 'category']).columns.tolist()
+        datetime_cols = df.select_dtypes(include=['datetime64']).columns.tolist()
+        boolean_cols = df.select_dtypes(include=['bool']).columns.tolist()
+        
+        self.validation_report["column_types"] = {
+            "numeric": numeric_cols,
+            "categorical": categorical_cols,
+            "datetime": datetime_cols,
+            "boolean": boolean_cols
+        }
+        
+        # Constant/near-constant columns
+        constant_cols = []
+        near_constant_cols = []
+        
+        for col in df.columns:
+            nunique = df[col].nunique()
+            if nunique == 1:
+                constant_cols.append(col)
+            elif nunique <= 2 and len(df) > 100:
+                near_constant_cols.append(col)
+        
+        self.validation_report["low_variance_columns"] = {
+            "constant": constant_cols,
+            "near_constant": near_constant_cols
+        }
+        
+        # Potential ID columns (high cardinality)
+        potential_id_cols = []
+        for col in df.columns:
+            if df[col].nunique() == len(df):
+                potential_id_cols.append(col)
+        
+        self.validation_report["potential_id_columns"] = potential_id_cols
+        
+        return self.validation_report
+    
+    def print_summary(self) -> None:
+        """Print a formatted summary of the validation report."""
+        if not self.validation_report:
+            self.validate()
+        
+        report = self.validation_report
+        
+        print("=" * 60)
+        logger.info("DATASET VALIDATION SUMMARY")
+        print("=" * 60)
+        
+        logger.info(f"\n📊 Shape: {report['shape'][0]:,} rows × {report['shape'][1]} columns")
+        logger.info(f"💾 Memory Usage: {report['memory_usage_mb']:.2f} MB")
+        
+        logger.info("\n📋 Column Types:")
+        for ctype, cols in report['column_types'].items():
+            if cols:
+                logger.info(f"   • {ctype.capitalize()}: {len(cols)} columns")
+        
+        missing = report['missing_values']
+        if missing['columns_with_missing'] > 0:
+            logger.info("\n⚠️  Missing Values:")
+            logger.info(f"   • Columns affected: {missing['columns_with_missing']}")
+            logger.info(f"   • Total missing cells: {missing['total_missing_cells']:,}")
+            for col, pct in list(missing['percentages'].items())[:5]:
+                logger.info(f"   • {col}: {pct}%")
+            if len(missing['percentages']) > 5:
+                logger.info(f"   ... and {len(missing['percentages']) - 5} more columns")
+        else:
+            logger.info("\n✅ No missing values detected")
+        
+        dups = report['duplicates']
+        if dups['count'] > 0:
+            logger.info(f"\n⚠️  Duplicates: {dups['count']:,} rows ({dups['percentage']}%)")
+        else:
+            logger.info("\n✅ No duplicate rows detected")
+        
+        if report['low_variance_columns']['constant']:
+            logger.info(f"\n⚠️  Constant columns: {report['low_variance_columns']['constant']}")
+        
+        if report['potential_id_columns']:
+            logger.info(f"\n🔑 Potential ID columns: {report['potential_id_columns']}")
+        
+        print("\n" + "=" * 60)

dataclean/drift_detector.py

@@ -0,0 +1,111 @@
+"""
+Data Drift Detector
+===================
+Detects schema changes and statistical drift between two datasets.
+"""
+
+import logging
+logger = logging.getLogger(__name__)
+import pandas as pd
+import numpy as np
+from scipy import stats
+from typing import Dict, Any, List, Optional
+
+class DriftDetector:
+    """
+    Detects data drift between a baseline dataset and a current dataset.
+    """
+    
+    def __init__(self, baseline_df: pd.DataFrame, current_df: pd.DataFrame):
+        self.baseline = baseline_df
+        self.current = current_df
+        self.report: Dict[str, Any] = {
+            'schema_drift': {},
+            'statistical_drift': {},
+            'score': 100
+        }
+    
+    def run(self) -> Dict[str, Any]:
+        """Run all drift checks."""
+        self._check_schema_drift()
+        self._check_statistical_drift()
+        self._calculate_score()
+        return self.report
+    
+    def _check_schema_drift(self):
+        """Check for missing or new columns."""
+        base_cols = set(self.baseline.columns)
+        curr_cols = set(self.current.columns)
+        
+        missing = list(base_cols - curr_cols)
+        new = list(curr_cols - base_cols)
+        
+        self.report['schema_drift'] = {
+            'missing_columns': missing,
+            'new_columns': new,
+            'has_drift': len(missing) > 0
+        }
+    
+    def _check_statistical_drift(self):
+        """
+        Check for statistical drift in shared numeric columns using KS-Test.
+        KS-Test (Kolmogorov-Smirnov) checks if two samples come from same distribution.
+        """
+        base_cols = set(self.baseline.select_dtypes(include=[np.number]).columns)
+        curr_cols = set(self.current.select_dtypes(include=[np.number]).columns)
+        shared_cols = list(base_cols.intersection(curr_cols))
+        
+        drifted_features = []
+        
+        for col in shared_cols:
+            # Drop NaNs for valid test
+            b_data = self.baseline[col].dropna()
+            c_data = self.current[col].dropna()
+            
+            if len(b_data) == 0 or len(c_data) == 0:
+                continue
+                
+            # KS Test
+            # Null hypothesis: samples are from same distribution.
+            # If p_value < 0.05, we reject null hypothesis -> DRIFT DETECTED.
+            stat, p_value = stats.ks_2samp(b_data, c_data)
+            
+            is_drifted = p_value < 0.05
+            
+            # Additional metrics
+            b_mean = b_data.mean()
+            c_mean = c_data.mean()
+            delta_mean = abs(b_mean - c_mean)
+            perc_change = (delta_mean / b_mean) * 100 if b_mean != 0 else 0
+            
+            self.report['statistical_drift'][col] = {
+                'p_value': float(p_value),
+                'is_drifted': bool(is_drifted),
+                'baseline_mean': float(b_mean),
+                'current_mean': float(c_mean),
+                'pct_change': float(perc_change)
+            }
+            
+            if is_drifted:
+                drifted_features.append(col)
+                
+        self.report['drifted_features'] = drifted_features
+        self.report['drift_detected'] = len(drifted_features) > 0
+
+    def _calculate_score(self):
+        """Calculate a simple health score (0-100)."""
+        # Penalty for schema drift
+        score = 100
+        if self.report['schema_drift']['has_drift']:
+            score -= 20
+        
+        # Penalty for statistical drift
+        total_feats = len(self.report['statistical_drift'])
+        drifted = len(self.report['drifted_features'])
+        
+        if total_feats > 0:
+            drift_ratio = drifted / total_feats
+            score -= (drift_ratio * 50)
+            
+        self.report['score'] = max(0, int(score))
+