data-advisor 0.1.0__py3-none-any.whl

data_advisor/codegen.py

@@ -0,0 +1,515 @@
+"""
+codegen.py - Generates copy-pasteable, robust, executable Python scripts for timed exams.
+
+Produces clean, PEP 8 compliant, production-grade Pandas, Scipy, and Scikit-Learn code
+tailored specifically to the audited dataset.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+from data_advisor.visualizer import VisualizationAdvisor
+
+
+class CodeGenerator:
+    """
+    Translates dataset scan and rule evaluations into sequential,
+    modular, executable Python code blocks and full pipeline scripts.
+    """
+
+    def __init__(self, scan_results: Dict[str, Any], data_source_str: str = "dataset.csv") -> None:
+        self.scan = scan_results
+        self.source = data_source_str
+        self.overview = scan_results.get("overview", {})
+        self.missing = scan_results.get("missing_values", {})
+        self.dtypes = scan_results.get("data_types", {})
+        self.distributions = scan_results.get("distributions", {})
+        self.categoricals = scan_results.get("categoricals", {})
+        self.target = scan_results.get("target")
+        self.hypothesis_tests = scan_results.get("hypothesis_tests", [])
+
+    # -------------------------------------------------------------------------
+    # 1. Environment & Ingestion Block
+    # -------------------------------------------------------------------------
+    def generate_imports_and_loading(self) -> str:
+        return f'''# =============================================================================
+# STEP 1: ENVIRONMENT SETUP & DATA INGESTION
+# =============================================================================
+import numpy as np
+import pandas as pd
+import scipy.stats as stats
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+from sklearn.model_selection import train_test_split, cross_val_score, KFold, StratifiedKFold
+from sklearn.preprocessing import StandardScaler, RobustScaler, OneHotEncoder, PowerTransformer
+from sklearn.impute import SimpleImputer, KNNImputer
+
+# Set presentation styles
+pd.set_option('display.max_columns', 50)
+pd.set_option('display.width', 1000)
+sns.set_theme(style='whitegrid', palette='muted')
+
+# Load the dataset
+DATA_PATH = {repr(self.source)}
+print(f"[*] Ingesting dataset from: {{DATA_PATH}}")
+df = pd.read_csv(DATA_PATH) if DATA_PATH.endswith('.csv') else pd.read_excel(DATA_PATH)
+
+print(f"[*] Raw Shape: {{df.shape[0]}} rows, {{df.shape[1]}} columns")
+df.info()
+'''
+
+    # -------------------------------------------------------------------------
+    # 2. Data Cleaning Block
+    # -------------------------------------------------------------------------
+    def generate_cleaning_code(self) -> str:
+        lines = [
+            "# =============================================================================",
+            "# STEP 2: DATA CLEANING & AUDITING",
+            "# =============================================================================",
+        ]
+
+        # Duplicate Handling
+        dup_count = self.overview.get("duplicate_rows", 0)
+        if dup_count > 0:
+            lines.extend([
+                f"# Remove {dup_count} duplicated rows",
+                "initial_rows = len(df)",
+                "df = df.drop_duplicates().reset_index(drop=True)",
+                "print(f'[*] Deduplicated dataset: dropped {initial_rows - len(df)} duplicate rows.')",
+                "",
+            ])
+        else:
+            lines.extend([
+                "# Verification: No duplicate rows detected",
+                "assert df.duplicated().sum() == 0, 'Unexpected duplicates found!'",
+                "",
+            ])
+
+        # Drop Constant Columns
+        constant_cols = self.dtypes.get("constant_columns", [])
+        if constant_cols:
+            lines.extend([
+                f"# Drop zero-variance (constant) columns: {constant_cols}",
+                f"df = df.drop(columns={repr(constant_cols)})",
+                f'print(f"[*] Dropped constant columns: {constant_cols}")',
+                "",
+            ])
+
+        # Drop High-Missing Columns (>60%)
+        high_missing_cols = [
+            c for c, info in self.missing.items() if info.get("null_pct", 0) >= 60.0
+        ]
+        if high_missing_cols:
+            lines.extend([
+                f"# Drop high-sparsity columns (>60% missing): {high_missing_cols}",
+                f"df = df.drop(columns={repr(high_missing_cols)})",
+                f'print(f"[*] Dropped sparse columns: {high_missing_cols}")',
+                "",
+            ])
+
+        # Date conversions & feature extraction
+        date_cols = self.dtypes.get("potential_dates", [])
+        if date_cols:
+            lines.extend([
+                "# Parse dates and extract temporal features",
+            ])
+            for d_col in date_cols:
+                lines.extend([
+                    f"df['{d_col}'] = pd.to_datetime(df['{d_col}'], errors='coerce')",
+                    f"df['{d_col}_year'] = df['{d_col}'].dt.year",
+                    f"df['{d_col}_month'] = df['{d_col}'].dt.month",
+                    f"df['{d_col}_day'] = df['{d_col}'].dt.day",
+                    f"df['{d_col}_dayofweek'] = df['{d_col}'].dt.dayofweek",
+                    f"df = df.drop(columns=['{d_col}'])  # Drop raw timestamp string",
+                ])
+            lines.append("")
+
+        # Missing Value Imputation
+        numeric_imputes_median = []
+        numeric_imputes_mean = []
+        cat_imputes_mode = []
+
+        for c, info in self.missing.items():
+            pct = info.get("null_pct", 0)
+            if 0 < pct < 60.0 and c not in high_missing_cols:
+                if info.get("is_numeric", False):
+                    skew = abs(self.distributions.get(c, {}).get("skewness", 0.0))
+                    if skew > 1.0:
+                        numeric_imputes_median.append(c)
+                    else:
+                        numeric_imputes_mean.append(c)
+                else:
+                    cat_imputes_mode.append(c)
+
+        if numeric_imputes_median or numeric_imputes_mean or cat_imputes_mode:
+            lines.append("# Missing Value Imputation")
+            if numeric_imputes_median:
+                lines.extend([
+                    f"# Median imputation for skewed numeric features: {numeric_imputes_median}",
+                    f"for col in {repr(numeric_imputes_median)}:",
+                    "    df[col] = df[col].fillna(df[col].median())",
+                ])
+            if numeric_imputes_mean:
+                lines.extend([
+                    f"# Mean imputation for normal numeric features: {numeric_imputes_mean}",
+                    f"for col in {repr(numeric_imputes_mean)}:",
+                    "    df[col] = df[col].fillna(df[col].mean())",
+                ])
+            if cat_imputes_mode:
+                lines.extend([
+                    f"# Mode imputation for categorical features: {cat_imputes_mode}",
+                    f"for col in {repr(cat_imputes_mode)}:",
+                    "    mode_val = df[col].mode()[0] if not df[col].mode().empty else 'Missing'",
+                    "    df[col] = df[col].fillna(mode_val)",
+                ])
+            lines.extend([
+                "print(f'[*] Remaining missing values: {df.isna().sum().sum()}')",
+                "",
+            ])
+
+        return "\n".join(lines)
+
+    # -------------------------------------------------------------------------
+    # 3. Distribution Transformations & Outlier Code
+    # -------------------------------------------------------------------------
+    def generate_transformations_code(self) -> str:
+        lines = [
+            "# =============================================================================",
+            "# STEP 3: DISTRIBUTION TRANSFORMATIONS & OUTLIER HANDLING",
+            "# =============================================================================",
+        ]
+
+        has_transforms = False
+        for col, dist in self.distributions.items():
+            skew = dist.get("skewness", 0.0)
+            is_non_neg = dist.get("is_non_negative", True)
+            iqr_pct = dist.get("iqr_outlier_pct", 0.0)
+            lf = dist.get("lower_fence", 0.0)
+            uf = dist.get("upper_fence", 0.0)
+
+            # High skewness
+            if skew > 1.2 and is_non_neg:
+                has_transforms = True
+                lines.extend([
+                    f"# Log1p transform for right-skewed column '{col}' (skewness={skew:.2f})",
+                    f"df['{col}'] = np.log1p(df['{col}'])",
+                ])
+            elif abs(skew) > 1.2 and not is_non_neg:
+                has_transforms = True
+                lines.extend([
+                    f"# Yeo-Johnson transform for skewed column '{col}' containing negative values",
+                    "pt = PowerTransformer(method='yeo-johnson')",
+                    f"df['{col}'] = pt.fit_transform(df[['{col}']])",
+                ])
+
+            # Extreme outliers clipping
+            if iqr_pct > 5.0:
+                has_transforms = True
+                lines.extend([
+                    f"# Winsorize/Clip extreme outliers for '{col}' at IQR fences [{lf:.2f}, {uf:.2f}]",
+                    f"df['{col}'] = df['{col}'].clip(lower={lf}, upper={uf})",
+                ])
+
+        if not has_transforms:
+            lines.append("# No severe skewness or extreme outliers requiring destructive transformation.\n")
+        else:
+            lines.append("print('[*] Distribution normalization and outlier treatment applied.')\n")
+
+        return "\n".join(lines)
+
+    # -------------------------------------------------------------------------
+    # 4. Categorical Encoding Block
+    # -------------------------------------------------------------------------
+    def generate_encoding_code(self) -> str:
+        lines = [
+            "# =============================================================================",
+            "# STEP 4: CATEGORICAL FEATURE ENCODING",
+            "# =============================================================================",
+        ]
+
+        target_col = self.target.get("target_column") if self.target else None
+        excluded_cols = set(
+            list(self.dtypes.get("potential_dates", []))
+            + list(self.dtypes.get("constant_columns", []))
+            + list(self.dtypes.get("potential_ids", []))
+            + [c for c, info in self.missing.items() if info.get("null_pct", 0) >= 60.0]
+        )
+        if target_col:
+            excluded_cols.add(target_col)
+
+        binary_cols = []
+        low_card_cols = []
+        high_card_cols = []
+
+        for col, cat in self.categoricals.items():
+            if col in excluded_cols:
+                continue
+            cardinality = cat.get("cardinality", 0)
+            if cardinality == 2:
+                binary_cols.append(col)
+            elif 3 <= cardinality <= 10:
+                low_card_cols.append(col)
+            elif cardinality > 10:
+                high_card_cols.append(col)
+
+        if binary_cols:
+            lines.extend([
+                f"# Binary 0/1 Encoding for binary features: {binary_cols}",
+                f"for col in {repr(binary_cols)}:",
+                "    if col in df.columns:",
+                "        unique_vals = df[col].dropna().unique()",
+                "        if len(unique_vals) == 2:",
+                "            val_map = {unique_vals[0]: 0, unique_vals[1]: 1}",
+                "            df[col] = df[col].map(val_map)",
+                '            print(f"[*] Mapped {col}: {val_map}")',
+                "",
+            ])
+
+        if low_card_cols:
+            lines.extend([
+                f"# One-Hot Encoding for low-cardinality features: {low_card_cols}",
+                f"ohe_cols = [c for c in {repr(low_card_cols)} if c in df.columns]",
+                "if ohe_cols:",
+                "    df = pd.get_dummies(df, columns=ohe_cols, drop_first=True, dtype=int)",
+                "    print(f'[*] Applied One-Hot Encoding on: {ohe_cols}')",
+                "",
+            ])
+
+        if high_card_cols:
+            lines.extend([
+                f"# Frequency Encoding for high-cardinality features: {high_card_cols}",
+                f"for col in {repr(high_card_cols)}:",
+                "    if col in df.columns:",
+                "        freq_encoding = df[col].value_counts(normalize=True)",
+                "        df[col] = df[col].map(freq_encoding)",
+                '        print(f"[*] Applied Frequency Encoding on {col}")',
+                "",
+            ])
+
+        if not (binary_cols or low_card_cols or high_card_cols):
+            lines.append("# No categorical columns requiring encoding.\n")
+
+        return "\n".join(lines)
+
+    # -------------------------------------------------------------------------
+    # 5. Hypothesis Testing Block
+    # -------------------------------------------------------------------------
+    def generate_hypothesis_testing_code(self) -> str:
+        lines = [
+            "# =============================================================================",
+            "# STEP 5: HYPOTHESIS TESTING & STATISTICAL INFERENCE",
+            "# =============================================================================",
+            "print('\\n' + '='*70)",
+            "print(' STATISTICAL HYPOTHESIS TESTING RESULTS')",
+            "print('='*70)",
+            "",
+        ]
+
+        if not self.hypothesis_tests:
+            lines.append("# No automated hypothesis tests generated.")
+            return "\n".join(lines)
+
+        for idx, test in enumerate(self.hypothesis_tests, 1):
+            f1 = test.get("feature_1")
+            f2 = test.get("feature_2")
+            test_type = test.get("test_type")
+            h0 = test.get("h0")
+            h1 = test.get("h1")
+
+            lines.extend([
+                f"# Test {idx}: {test_type} ({f1} vs {f2})",
+                f"if {repr(f1)} in df.columns and {repr(f2)} in df.columns:",
+                f"    print('\\n' + '-'*50)",
+                f"    print({repr(f'[TEST {idx}] {test_type}: {f1} vs {f2}')})",
+                f"    print({repr(f'H0: {h0}')})",
+                f"    print({repr(f'H1: {h1}')})",
+            ])
+
+            if "Correlation" in test_type:
+                lines.extend([
+                    f"    valid_data = df[[{repr(f1)}, {repr(f2)}]].dropna()",
+                    f"    r, p_val = stats.pearsonr(valid_data[{repr(f1)}], valid_data[{repr(f2)}])",
+                    f"    rho, spearman_pval = stats.spearmanr(valid_data[{repr(f1)}], valid_data[{repr(f2)}])",
+                    "    print(f'Pearson r: {r:.4f} (p-value: {p_val:.4e})')",
+                    "    print(f'Spearman rho: {rho:.4f} (p-value: {spearman_pval:.4e})')",
+                    "    if p_val < 0.05:",
+                    "        print('=> CONCLUSION: Reject H0. Statistically significant correlation detected.')",
+                    "    else:",
+                    "        print('=> CONCLUSION: Fail to reject H0. No significant linear correlation.')",
+                ])
+            elif "t-test" in test_type or "Mann-Whitney" in test_type:
+                lines.extend([
+                    f"    valid_data = df[[{repr(f1)}, {repr(f2)}]].dropna()",
+                    f"    classes = valid_data[{repr(f2)}].unique()",
+                    f"    if len(classes) >= 2:",
+                    f"        g1 = valid_data[valid_data[{repr(f2)}] == classes[0]][{repr(f1)}]",
+                    f"        g2 = valid_data[valid_data[{repr(f2)}] == classes[1]][{repr(f1)}]",
+                    "        t_stat, p_val = stats.ttest_ind(g1, g2, equal_var=False)",
+                    "        u_stat, u_pval = stats.mannwhitneyu(g1, g2, alternative='two-sided')",
+                    "        print(f'Welch t-test: t={t_stat:.4f}, p-value={p_val:.4e}')",
+                    "        print(f'Mann-Whitney U: U={u_stat:.4f}, p-value={u_pval:.4e}')",
+                    "        if p_val < 0.05:",
+                    "            print('=> CONCLUSION: Reject H0. Significant difference between class distributions.')",
+                    "        else:",
+                    "            print('=> CONCLUSION: Fail to reject H0. No significant difference.')",
+                ])
+            elif "ANOVA" in test_type:
+                lines.extend([
+                    f"    valid_data = df[[{repr(f1)}, {repr(f2)}]].dropna()",
+                    f"    classes = valid_data[{repr(f2)}].unique()",
+                    f"    groups = [valid_data[valid_data[{repr(f2)}] == c][{repr(f1)}] for c in classes]",
+                    "    f_stat, p_val = stats.f_oneway(*groups)",
+                    "    print(f'One-way ANOVA: F={f_stat:.4f}, p-value={p_val:.4e}')",
+                    "    if p_val < 0.05:",
+                    "        print('=> CONCLUSION: Reject H0. Group means differ significantly.')",
+                    "    else:",
+                    "        print('=> CONCLUSION: Fail to reject H0. No significant difference in group means.')",
+                ])
+            elif "Chi-Square" in test_type:
+                lines.extend([
+                    f"    valid_data = df[[{repr(f1)}, {repr(f2)}]].dropna()",
+                    f"    contingency_tab = pd.crosstab(valid_data[{repr(f1)}], valid_data[{repr(f2)}])",
+                    "    chi2, p_val, dof, _ = stats.chi2_contingency(contingency_tab)",
+                    "    n_obs = contingency_tab.sum().sum()",
+                    "    min_dim = min(contingency_tab.shape) - 1",
+                    "    cramer_v = np.sqrt(chi2 / (n_obs * min_dim)) if (n_obs * min_dim) > 0 else 0.0",
+                    "    print(f'Chi-Square: chi2={chi2:.4f}, dof={dof}, p-value={p_val:.4e}')",
+                    "    print(f'Cramer V Effect Size: {cramer_v:.4f}')",
+                    "    if p_val < 0.05:",
+                    "        print('=> CONCLUSION: Reject H0. Significant association/dependency between categorical features.')",
+                    "    else:",
+                    "        print('=> CONCLUSION: Fail to reject H0. Variables appear statistically independent.')",
+                ])
+            lines.append("")
+
+        return "\n".join(lines)
+
+    # -------------------------------------------------------------------------
+    # 6. Baseline Modeling Block
+    # -------------------------------------------------------------------------
+    def generate_modeling_code(self) -> str:
+        if not self.target:
+            return "# No target variable detected; modeling pipeline skipped.\n"
+
+        target_col = self.target.get("target_column")
+        prob_type = self.target.get("problem_type")
+        id_cols = list(self.dtypes.get("potential_ids", []))
+
+        lines = [
+            "# =============================================================================",
+            f"# STEP 6: MACHINE LEARNING PIPELINE ({prob_type.upper()})",
+            "# =============================================================================",
+            f"TARGET = {repr(target_col)}",
+            f"ID_COLS = {repr(id_cols)}",
+            "",
+            "# Separate Feature Matrix (X) and Response Vector (y)",
+            "drop_cols = [TARGET] + [col for col in ID_COLS if col in df.columns]",
+            "X = df.drop(columns=[col for col in drop_cols if col in df.columns])",
+            "y = df[TARGET]",
+            "",
+            "# Align X with only numeric types (already encoded)",
+            "X = X.select_dtypes(include=[np.number])",
+            "print(f'[*] Modeling Matrix: X shape = {X.shape}, y shape = {y.shape}')",
+            "",
+        ]
+
+        if prob_type == "regression":
+            lines.extend([
+                "from sklearn.linear_model import LinearRegression, Ridge",
+                "from sklearn.ensemble import RandomForestRegressor",
+                "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score",
+                "",
+                "# Train / Test Split (80/20)",
+                "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42)",
+                "",
+                "# Feature Scaling with RobustScaler (robust to remaining outliers)",
+                "scaler = RobustScaler()",
+                "X_train_scaled = scaler.fit_transform(X_train)",
+                "X_test_scaled = scaler.transform(X_test)",
+                "",
+                "# Model Baselines",
+                "models = {",
+                "    'Linear Regression': LinearRegression(),",
+                "    'Ridge Regression (L2)': Ridge(alpha=1.0),",
+                "    'Random Forest Regressor': RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1),",
+                "}",
+                "",
+                "print('\\n' + '='*75)",
+                "print(f'{\"Model\":<25} | {\"CV R2 (5-Fold)\":<15} | {\"Test RMSE\":<12} | {\"Test R2\":<10}')",
+                "print('='*75)",
+                "",
+                "for name, model in models.items():",
+                "    # 5-Fold Cross Validation on Training set",
+                "    cv_scores = cross_val_score(model, X_train_scaled, y_train, cv=5, scoring='r2')",
+                "    # Train on full train set and evaluate on test set",
+                "    model.fit(X_train_scaled, y_train)",
+                "    preds = model.predict(X_test_scaled)",
+                "    # Calculate metrics",
+                "    rmse = np.sqrt(mean_squared_error(y_test, preds))",
+                "    r2 = r2_score(y_test, preds)",
+                "    mae = mean_absolute_error(y_test, preds)",
+                "    print(f'{name:<25} | {cv_scores.mean():.4f} (+/-{cv_scores.std():.3f}) | {rmse:<12.4f} | {r2:<10.4f}')",
+                "print('='*75)",
+            ])
+        else:
+            # Classification
+            is_imbalanced = self.target.get("is_imbalanced", False)
+            cw_param = "'balanced'" if is_imbalanced else "None"
+
+            lines.extend([
+                "from sklearn.linear_model import LogisticRegression",
+                "from sklearn.ensemble import RandomForestClassifier",
+                "from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score",
+                "",
+                "# Stratified Train / Test Split (preserves class distribution)",
+                "X_train, X_test, y_train, y_test = train_test_split(",
+                "    X, y, test_size=0.20, random_state=42, stratify=y",
+                ")",
+                "",
+                "# Feature Scaling",
+                "scaler = StandardScaler()",
+                "X_train_scaled = scaler.fit_transform(X_train)",
+                "X_test_scaled = scaler.transform(X_test)",
+                "",
+                "# Classification Models",
+                "models = {",
+                f"    'Logistic Regression': LogisticRegression(class_weight={cw_param}, max_iter=1000, random_state=42),",
+                f"    'Random Forest Classifier': RandomForestClassifier(n_estimators=100, class_weight={cw_param}, random_state=42, n_jobs=-1),",
+                "}",
+                "",
+                "cv_strat = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)",
+                "",
+                "for name, model in models.items():",
+                "    print(f'\\n>> Evaluating {name}:')",
+                "    # 5-Fold Stratified Cross Validation",
+                "    cv_scores = cross_val_score(model, X_train_scaled, y_train, cv=cv_strat, scoring='f1_weighted')",
+                "    print(f'   5-Fold CV Weighted F1: {cv_scores.mean():.4f} (+/- {cv_scores.std():.3f})')",
+                "    # Fit and Predict on Test Set",
+                "    model.fit(X_train_scaled, y_train)",
+                "    preds = model.predict(X_test_scaled)",
+                "    print('   Test Set Classification Report:')",
+                "    print(classification_report(y_test, preds))",
+                "    print('   Confusion Matrix:')",
+                "    print(confusion_matrix(y_test, preds))",
+            ])
+
+        return "\n".join(lines)
+
+    # -------------------------------------------------------------------------
+    # 7. Master Script Assembly
+    # -------------------------------------------------------------------------
+    def generate_full_script(self) -> str:
+        """Assemble all modular blocks into one comprehensive, executable script."""
+        vis_advisor = VisualizationAdvisor(self.scan)
+        script_parts = [
+            self.generate_imports_and_loading(),
+            self.generate_cleaning_code(),
+            self.generate_transformations_code(),
+            self.generate_encoding_code(),
+            vis_advisor.generate_all_plots_code(),
+            self.generate_hypothesis_testing_code(),
+            self.generate_modeling_code(),
+            "\nprint('\\n[*] Complete Data Analysis & Modeling Pipeline Finished Successfully.')\n",
+        ]
+        return "\n\n".join(script_parts)