data-advisor 0.1.0__py3-none-any.whl

data_advisor/rules.py

@@ -0,0 +1,380 @@
+"""
+rules.py - Heuristic decision engine for automated data diagnostics and modeling strategy.
+
+Translates raw profiling metrics into high-impact, exam-winning recommendations
+for cleaning, transformations, feature engineering, and baseline modeling.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class Recommendation:
+    """Standardized recommendation container."""
+    category: str        # 'cleaning', 'transformation', 'encoding', 'modeling', 'testing'
+    severity: str        # 'CRITICAL', 'WARNING', 'RECOMMENDED', 'INFO'
+    column: Optional[str]
+    issue: str
+    action: str
+    code_hint: str
+    rationale: str
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+
+class RuleEngine:
+    """
+    Evaluates scan results against industry-standard data science heuristics
+    and produces prioritized, actionable advice.
+    """
+
+    def __init__(self, scan_results: Dict[str, Any]) -> None:
+        self.scan = scan_results
+        self.overview = scan_results.get("overview", {})
+        self.missing = scan_results.get("missing_values", {})
+        self.dtypes = scan_results.get("data_types", {})
+        self.distributions = scan_results.get("distributions", {})
+        self.categoricals = scan_results.get("categoricals", {})
+        self.target = scan_results.get("target")
+        self.hypothesis_tests = scan_results.get("hypothesis_tests", [])
+
+    # -------------------------------------------------------------------------
+    # 1. Cleaning & Auditing Rules
+    # -------------------------------------------------------------------------
+    def evaluate_cleaning_rules(self) -> List[Recommendation]:
+        recs: List[Recommendation] = []
+
+        # Duplicate Rows
+        dup_count = self.overview.get("duplicate_rows", 0)
+        dup_pct = self.overview.get("duplicate_pct", 0.0)
+        if dup_count > 0:
+            severity = "CRITICAL" if dup_pct > 5.0 else "WARNING"
+            recs.append(Recommendation(
+                category="cleaning",
+                severity=severity,
+                column=None,
+                issue=f"Found {dup_count} duplicate rows ({dup_pct}% of dataset).",
+                action="Deduplicate dataset before analysis or modeling.",
+                code_hint="df = df.drop_duplicates().reset_index(drop=True)",
+                rationale="Duplicate rows cause artificial data leakage and bias model evaluations.",
+            ))
+
+        # Constant Columns
+        for col in self.dtypes.get("constant_columns", []):
+            recs.append(Recommendation(
+                category="cleaning",
+                severity="WARNING",
+                column=col,
+                issue=f"Column '{col}' has zero variance (constant value).",
+                action=f"Drop column '{col}'.",
+                code_hint=f"df = df.drop(columns=['{col}'])",
+                rationale="Zero variance features provide zero predictive signal and cause multicollinearity.",
+            ))
+
+        # High-Risk / High-Cardinality ID Columns
+        for col in self.dtypes.get("potential_ids", []):
+            recs.append(Recommendation(
+                category="cleaning",
+                severity="WARNING",
+                column=col,
+                issue=f"Column '{col}' appears to be a unique identifier / primary key.",
+                action=f"Exclude '{col}' from model training features.",
+                code_hint=f"id_col = '{col}'\ndf_features = df.drop(columns=['{col}'])",
+                rationale="Identifiers cause spurious correlation, extreme overfitting, and data leakage.",
+            ))
+
+        # Missing Value Strategies
+        for col, info in self.missing.items():
+            pct = info.get("null_pct", 0.0)
+            count = info.get("null_count", 0)
+            is_num = info.get("is_numeric", False)
+
+            if pct == 0.0:
+                continue
+
+            if pct >= 60.0:
+                recs.append(Recommendation(
+                    category="cleaning",
+                    severity="CRITICAL",
+                    column=col,
+                    issue=f"Column '{col}' has {pct}% missing values ({count} rows).",
+                    action=f"Drop column '{col}' due to high sparsity (>60%).",
+                    code_hint=f"df = df.drop(columns=['{col}'])",
+                    rationale="Imputing >60% missing data injects massive noise and distorts distributions.",
+                ))
+            elif pct < 5.0:
+                action_text = "Impute with median or drop missing rows." if is_num else "Impute with mode or drop missing rows."
+                code_text = (
+                    f"df['{col}'] = df['{col}'].fillna(df['{col}'].median())"
+                    if is_num
+                    else f"df['{col}'] = df['{col}'].fillna(df['{col}'].mode()[0])"
+                )
+                recs.append(Recommendation(
+                    category="cleaning",
+                    severity="INFO",
+                    column=col,
+                    issue=f"Column '{col}' has minor missing values: {pct}% ({count} rows).",
+                    action=action_text,
+                    code_hint=code_text,
+                    rationale="Low missingness (<5%) can be safely imputed with median/mode without altering distribution shape.",
+                ))
+            else:
+                # 5% to 60% missing
+                if is_num:
+                    dist = self.distributions.get(col, {})
+                    skew = abs(dist.get("skewness", 0.0))
+                    strategy = "median" if skew > 1.0 else "mean"
+                    recs.append(Recommendation(
+                        category="cleaning",
+                        severity="WARNING",
+                        column=col,
+                        issue=f"Column '{col}' has moderate missing values: {pct}% ({count} rows).",
+                        action=f"Impute with {strategy} (or use KNNImputer).",
+                        code_hint=f"from sklearn.impute import SimpleImputer\nimputer = SimpleImputer(strategy='{strategy}')\ndf['{col}'] = imputer.fit_transform(df[['{col}']])",
+                        rationale=f"Skewness is {dist.get('skewness', 0.0):.2f}. Median is robust to skew; mean is suitable for symmetric data.",
+                    ))
+                else:
+                    recs.append(Recommendation(
+                        category="cleaning",
+                        severity="WARNING",
+                        column=col,
+                        issue=f"Categorical column '{col}' has {pct}% missing values.",
+                        action="Impute with mode or treat missingness as an explicit category ('Unknown').",
+                        code_hint=f"df['{col}'] = df['{col}'].fillna('Missing')",
+                        rationale="Preserving missingness as 'Missing' allows models to capture potential informative missingness bias.",
+                    ))
+
+        # Date string mismatches
+        for col in self.dtypes.get("potential_dates", []):
+            recs.append(Recommendation(
+                category="cleaning",
+                severity="RECOMMENDED",
+                column=col,
+                issue=f"Column '{col}' contains date-like strings but is stored as object/string.",
+                action=f"Convert '{col}' to datetime and extract temporal components.",
+                code_hint=(
+                    f"df['{col}'] = pd.to_datetime(df['{col}'], errors='coerce')\n"
+                    f"df['{col}_year'] = df['{col}'].dt.year\n"
+                    f"df['{col}_month'] = df['{col}'].dt.month\n"
+                    f"df['{col}_dayofweek'] = df['{col}'].dt.dayofweek"
+                ),
+                rationale="Machine learning models cannot process raw date strings; cyclic/temporal features extract high predictive value.",
+            ))
+
+        return recs
+
+    # -------------------------------------------------------------------------
+    # 2. Distribution & Outlier Transformation Rules
+    # -------------------------------------------------------------------------
+    def evaluate_distribution_rules(self) -> List[Recommendation]:
+        recs: List[Recommendation] = []
+
+        for col, dist in self.distributions.items():
+            skew = dist.get("skewness", 0.0)
+            kurt = dist.get("kurtosis", 0.0)
+            is_non_neg = dist.get("is_non_negative", True)
+            iqr_pct = dist.get("iqr_outlier_pct", 0.0)
+            iqr_cnt = dist.get("iqr_outliers", 0)
+            lower_fence = dist.get("lower_fence", 0.0)
+            upper_fence = dist.get("upper_fence", 0.0)
+
+            # Heavy Positive Skewness
+            if skew > 1.2:
+                if is_non_neg:
+                    recs.append(Recommendation(
+                        category="transformation",
+                        severity="RECOMMENDED",
+                        column=col,
+                        issue=f"Column '{col}' is heavily right-skewed (skewness = {skew:.2f}).",
+                        action="Apply log1p transformation to normalize distribution.",
+                        code_hint=f"df['{col}_log'] = np.log1p(df['{col}'])",
+                        rationale="Log transformation stabilizes variance and minimizes leverage of extreme positive values in linear/distance models.",
+                    ))
+                else:
+                    recs.append(Recommendation(
+                        category="transformation",
+                        severity="RECOMMENDED",
+                        column=col,
+                        issue=f"Column '{col}' is right-skewed (skewness = {skew:.2f}) and contains negative values.",
+                        action="Apply Yeo-Johnson power transformation.",
+                        code_hint=(
+                            f"from sklearn.preprocessing import PowerTransformer\n"
+                            f"pt = PowerTransformer(method='yeo-johnson')\n"
+                            f"df['{col}_transformed'] = pt.fit_transform(df[['{col}']])"
+                        ),
+                        rationale="Yeo-Johnson supports zero and negative values while normalizing non-Gaussian distributions.",
+                    ))
+            elif skew < -1.2:
+                recs.append(Recommendation(
+                    category="transformation",
+                    severity="RECOMMENDED",
+                    column=col,
+                    issue=f"Column '{col}' is heavily left-skewed (skewness = {skew:.2f}).",
+                    action="Apply Yeo-Johnson transformation or upper/lower bound clipping.",
+                    code_hint=(
+                        f"from sklearn.preprocessing import PowerTransformer\n"
+                        f"pt = PowerTransformer(method='yeo-johnson')\n"
+                        f"df['{col}_transformed'] = pt.fit_transform(df[['{col}']])"
+                    ),
+                    rationale="Left-skewed distributions violate normality assumptions in parametric tests and models.",
+                ))
+
+            # Severe Outliers
+            if iqr_pct > 3.0:
+                recs.append(Recommendation(
+                    category="transformation",
+                    severity="WARNING",
+                    column=col,
+                    issue=f"Column '{col}' has {iqr_cnt} IQR outliers ({iqr_pct}%).",
+                    action=f"Clip values to IQR fences [{lower_fence:.2f}, {upper_fence:.2f}] or scale with RobustScaler.",
+                    code_hint=(
+                        f"df['{col}_clipped'] = df['{col}'].clip(lower={lower_fence}, upper={upper_fence})"
+                    ),
+                    rationale="Extreme outliers distort mean, inflate standard error, and destabilize gradient-based estimators.",
+                ))
+
+        return recs
+
+    # -------------------------------------------------------------------------
+    # 3. Categorical Processing Rules
+    # -------------------------------------------------------------------------
+    def evaluate_categorical_rules(self) -> List[Recommendation]:
+        recs: List[Recommendation] = []
+
+        for col, cat in self.categoricals.items():
+            cardinality = cat.get("cardinality", 0)
+
+            if cardinality == 2:
+                samples = cat.get("sample_values", [])
+                recs.append(Recommendation(
+                    category="encoding",
+                    severity="RECOMMENDED",
+                    column=col,
+                    issue=f"Binary categorical column '{col}' (values: {samples}).",
+                    action="Map directly to binary 0/1 integers.",
+                    code_hint=(
+                        f"mapping = {{{repr(samples[0])}: 0, {repr(samples[1])}: 1}}\n"
+                        f"df['{col}_encoded'] = df['{col}'].map(mapping)"
+                    ),
+                    rationale="Binary integer mapping avoids redundant dimensions while maintaining zero information loss.",
+                ))
+            elif 3 <= cardinality <= 10:
+                recs.append(Recommendation(
+                    category="encoding",
+                    severity="RECOMMENDED",
+                    column=col,
+                    issue=f"Low-cardinality categorical column '{col}' ({cardinality} levels).",
+                    action="Use One-Hot Encoding with drop_first=True.",
+                    code_hint=f"df = pd.get_dummies(df, columns=['{col}'], drop_first=True, dtype=int)",
+                    rationale="One-hot encoding with drop_first prevents the dummy variable trap (multicollinearity).",
+                ))
+            elif cardinality > 10:
+                recs.append(Recommendation(
+                    category="encoding",
+                    severity="WARNING",
+                    column=col,
+                    issue=f"High-cardinality categorical column '{col}' ({cardinality} unique levels).",
+                    action="Apply Frequency Encoding, Target Encoding, or collapse rare categories into 'Other'.",
+                    code_hint=(
+                        f"# Frequency Encoding\n"
+                        f"freq_map = df['{col}'].value_counts(normalize=True)\n"
+                        f"df['{col}_freq'] = df['{col}'].map(freq_map)\n\n"
+                        f"# Or Scikit-Learn Target Encoding\n"
+                        f"# from sklearn.preprocessing import TargetEncoder\n"
+                        f"# te = TargetEncoder(smooth='auto')\n"
+                        f"# df['{col}_te'] = te.fit_transform(df[['{col}']], y)"
+                    ),
+                    rationale="One-hot encoding high-cardinality columns explodes feature space, causing the curse of dimensionality.",
+                ))
+
+        return recs
+
+    # -------------------------------------------------------------------------
+    # 4. Target Variable & Baseline Modeling Strategy
+    # -------------------------------------------------------------------------
+    def evaluate_modeling_rules(self) -> List[Recommendation]:
+        recs: List[Recommendation] = []
+        if not self.target:
+            return recs
+
+        target_col = self.target.get("target_column")
+        prob_type = self.target.get("problem_type")
+
+        if prob_type == "regression":
+            recs.append(Recommendation(
+                category="modeling",
+                severity="INFO",
+                column=target_col,
+                issue=f"Target '{target_col}' is continuous ({self.target.get('n_samples')} valid samples).",
+                action="Deploy Regression Pipeline (Baseline Linear -> Ridge -> RandomForestRegressor).",
+                code_hint=(
+                    "from sklearn.linear_model import LinearRegression, Ridge\n"
+                    "from sklearn.ensemble import RandomForestRegressor\n"
+                    "from sklearn.metrics import root_mean_squared_error, mean_absolute_error, r2_score\n\n"
+                    "models = {\n"
+                    "    'LinearRegression': LinearRegression(),\n"
+                    "    'Ridge': Ridge(alpha=1.0),\n"
+                    "    'RandomForest': RandomForestRegressor(n_estimators=100, random_state=42)\n"
+                    "}"
+                ),
+                rationale="Provides both interpretable parametric baselines (OLS/Ridge) and robust non-linear tree ensembles.",
+            ))
+        elif prob_type == "classification":
+            is_imbalanced = self.target.get("is_imbalanced", False)
+            imbalance_ratio = self.target.get("imbalance_ratio", 1.0)
+            minority_pct = self.target.get("minority_percentage", 50.0)
+
+            severity = "CRITICAL" if is_imbalanced else "INFO"
+            issue_msg = (
+                f"Severe target imbalance detected! Minority class: {minority_pct}% (Ratio 1:{imbalance_ratio:.1f})."
+                if is_imbalanced
+                else f"Balanced classification target ({self.target.get('n_classes')} classes)."
+            )
+
+            action_msg = (
+                "Use StratifiedKFold, class_weight='balanced', and optimize for PR-AUC/F1-Score."
+                if is_imbalanced
+                else "Deploy Classification Pipeline (LogisticRegression -> RandomForestClassifier)."
+            )
+
+            code_hint = (
+                "from sklearn.linear_model import LogisticRegression\n"
+                "from sklearn.ensemble import RandomForestClassifier\n"
+                "from sklearn.metrics import classification_report, roc_auc_score\n\n"
+                "models = {\n"
+                f"    'LogisticRegression': LogisticRegression(class_weight={'\"balanced\"' if is_imbalanced else 'None'}, max_iter=1000),\n"
+                f"    'RandomForest': RandomForestClassifier(n_estimators=100, class_weight={'\"balanced\"' if is_imbalanced else 'None'}, random_state=42)\n"
+                "}"
+            )
+
+            recs.append(Recommendation(
+                category="modeling",
+                severity=severity,
+                column=target_col,
+                issue=issue_msg,
+                action=action_msg,
+                code_hint=code_hint,
+                rationale="Accuracy is deceptive under class imbalance. Balanced loss penalties and precision-recall metrics prevent majority-class collapse.",
+            ))
+
+        return recs
+
+    # -------------------------------------------------------------------------
+    # 5. Master Recommendation Engine
+    # -------------------------------------------------------------------------
+    def generate_all_recommendations(self) -> List[Recommendation]:
+        """Aggregate all rule checks in order of execution priority."""
+        recs: List[Recommendation] = []
+        recs.extend(self.evaluate_cleaning_rules())
+        recs.extend(self.evaluate_distribution_rules())
+        recs.extend(self.evaluate_categorical_rules())
+        recs.extend(self.evaluate_modeling_rules())
+
+        # Sort recommendations by severity priority: CRITICAL -> WARNING -> RECOMMENDED -> INFO
+        priority_map = {"CRITICAL": 0, "WARNING": 1, "RECOMMENDED": 2, "INFO": 3}
+        recs.sort(key=lambda r: priority_map.get(r.severity, 4))
+        return recs