feature-pruning 0.1.0__py3-none-any.whl

feature_pruning/__init__.py

@@ -0,0 +1,39 @@
+"""
+Feature Pruning: High-Performance Feature Reduction for Credit Risk Scorecards
+=============================================================================
+
+`feature_pruning` is an enterprise-grade, distributed feature reduction library
+specifically optimized for developing regulatory-compliant credit risk scorecards
+(such as Basel II/III, IFRS 9, and retail scorecard models).
+
+Key Capabilities:
+-----------------
+- Automated variable classification (numeric, categorical, datetime, high-cardinality).
+- Memory-efficient precision downcasting for massive distributed datasets.
+- Fast distributed Information Value (IV) calculation via quantile binning.
+- IV-prioritized Pearson correlation matrix pruning to eliminate multicollinearity.
+- Transparent audit trail tracking all feature inclusion and exclusion decisions.
+
+Quick Example:
+--------------
+>>> from feature_pruning import FeatureSelectionPipeline
+>>> pipeline = FeatureSelectionPipeline(
+...     df=spark_df,
+...     target_col="default_flag",
+...     mandatory_features=["bureau_score", "annual_income"],
+...     exclude_features=["applicant_id"],
+...     iv_threshold=0.03,
+...     corr_threshold=0.95,
+... )
+>>> pruned_spark_df, audit_report = pipeline.run()
+"""
+
+from .core import FeatureSelectionPipeline
+
+__version__ = "0.1.0"
+__author__ = "Vrukshya"
+__all__ = [
+    "FeatureSelectionPipeline",
+    "__version__",
+    "__author__",
+]

feature_pruning/core.py

@@ -0,0 +1,486 @@
+import time
+import warnings
+from typing import List, Tuple, Dict, Any, Optional
+from collections import defaultdict
+
+import numpy as np
+import pandas as pd
+import pyspark.sql.functions as F
+from pyspark.sql import SparkSession, DataFrame as SparkDataFrame
+from pyspark.sql.types import (
+    IntegerType, LongType, FloatType, DoubleType, DecimalType,
+    ByteType, ShortType, StringType, StructType, StructField,
+)
+from pyspark.storagelevel import StorageLevel
+from pyspark.ml.stat import Correlation
+from pyspark.ml.feature import VectorAssembler
+
+warnings.filterwarnings("ignore")
+
+
+class FeatureSelectionPipeline:
+    """End-to-end distributed feature reduction pipeline for credit risk scorecards.
+
+    This pipeline systematically filters high-dimensional feature spaces (e.g., thousands
+    of bureau, demographic, and transactional attributes) down to a curated, highly predictive,
+    and non-collinear feature set ready for Weight of Evidence (WoE) binning and Logistic
+    Regression scorecard modeling.
+
+    Execution Stages:
+        1. **step_identify_features**: Classifies columns into numeric, categorical,
+           datetime (auto-excluded), and high-cardinality (excluded if unique values > max_cat_levels).
+        2. **step_downcast**: Casts 64-bit numeric types (DecimalType, DoubleType) to FloatType
+           to reduce Spark cluster memory footprint and accelerate matrix computations.
+        3. **step_compute_iv**: Calculates Information Value (IV) across all candidates using
+           distributed quantile binning (`mapInPandas`). Features below `iv_threshold` are filtered out.
+        4. **step_prune_correlations**: Computes a Pearson correlation matrix via PySpark ML.
+           When two features have correlation >= `corr_threshold`, the feature with lower IV
+           is discarded to prevent multicollinearity.
+        5. **step_build_report**: Compiles a comprehensive audit DataFrame documenting the
+           status and exact exclusion reason for every input column.
+
+    Parameters:
+        df: Spark DataFrame containing candidate features and the binary target column.
+        target_col: Name of the binary target column (e.g., 'default_flag', 'bad_loan').
+            Must contain 0 (non-event / good) and 1 (event / bad).
+        mandatory_features: Optional list of column names that must never be dropped
+            regardless of IV or correlation (e.g., regulatory KPI ratios, bureau scores).
+        exclude_features: Optional list of column names to explicitly exclude from selection
+            (e.g., ID columns, leakage variables, date stamps).
+        iv_threshold: Minimum Information Value required to retain a feature.
+            Scorecard convention: < 0.02 (unpredictive), 0.02 - 0.1 (weak),
+            0.1 - 0.3 (medium), 0.3 - 0.5 (strong). Default is 0.03.
+        iv_n_bins: Number of quantile bins used for computing IV on continuous variables.
+            Default is 100.
+        iv_sample_rows: Maximum sample size used for computing distributed IV. Default is 1,000,000.
+        corr_threshold: Absolute Pearson correlation threshold above which collinear features
+            are pruned. Default is 0.95.
+        corr_sample_rows: Maximum sample size used for the correlation matrix calculation.
+            Default is 500,000.
+        max_corr_features: Maximum number of top IV features to feed into correlation pruning.
+            Default is 5000.
+        max_cat_levels: Maximum number of distinct categories permitted before a categorical
+            feature is flagged as high-cardinality and excluded. Default is 200.
+        expected_final_min: Minimum expected number of retained features. If correlation pruning
+            drops too many features, the threshold dynamically relaxes. Default is 200.
+        expected_final_max: Maximum cap on the final number of retained features. Default is 1000.
+        downcast: Whether to cast 64-bit floats/decimals to 32-bit float for memory efficiency.
+            Default is True.
+        verbose: Whether to log pipeline progress and execution times. Default is True.
+    """
+
+    def __init__(
+        self,
+        df: SparkDataFrame,
+        target_col: str,
+        mandatory_features: Optional[List[str]] = None,
+        exclude_features: Optional[List[str]] = None,
+        iv_threshold: float = 0.03,
+        iv_n_bins: int = 100,
+        iv_sample_rows: int = 1_000_000,
+        corr_threshold: float = 0.95,
+        corr_sample_rows: int = 500_000,
+        max_corr_features: int = 5000,
+        max_cat_levels: int = 200,
+        expected_final_min: int = 200,
+        expected_final_max: int = 1000,
+        downcast: bool = True,
+        verbose: bool = True,
+    ):
+        self._log = print if verbose else (lambda *a, **k: None)
+        self.target_col = target_col.lower()
+        self.mandatory_features = [f.lower() for f in (mandatory_features or [])]
+        self.exclude_features = [f.lower() for f in (exclude_features or [])]
+        self.iv_threshold = iv_threshold
+        self.iv_n_bins = iv_n_bins
+        self.iv_sample_rows = iv_sample_rows
+        self.corr_threshold = corr_threshold
+        self.corr_sample_rows = corr_sample_rows
+        self.max_corr_features = max_corr_features
+        self.max_cat_levels = max_cat_levels
+        self.expected_final_min = expected_final_min
+        self.expected_final_max = expected_final_max
+        self.downcast_enabled = downcast
+
+        self.df_spark_ = df.toDF(*[c.lower() for c in df.columns])
+        self._spark = df.sparkSession
+
+        self.feature_types_ = None
+        self.iv_results_ = None
+        self.correlation_drops_ = None
+        self.final_features_ = None
+        self.feature_report_ = None
+        self.result_df_ = None
+        self._n_rows = None
+        self._direct_ok = None
+        self._df_work = None
+
+    def step_identify_features(self):
+        self._log("\n[Step 1/5] Identifying feature types...")
+        dtypes_str = dict(self.df_spark_.dtypes)
+        all_cols = [c for c in self.df_spark_.columns if c != self.target_col]
+        exclude_set = set(self.exclude_features)
+        mandatory_set = set(self.mandatory_features)
+        cat_type_names = {"string", "boolean"}
+        dt_type_names = {"date", "timestamp"}
+
+        numeric_cols, cat_candidates, datetime_cols = [], [], []
+        for c in all_cols:
+            if c in exclude_set:
+                continue
+            if dtypes_str[c] in dt_type_names:
+                datetime_cols.append(c)
+            elif dtypes_str[c] in cat_type_names:
+                cat_candidates.append(c)
+            else:
+                numeric_cols.append(c)
+
+        high_cardinality, keep_cat = [], []
+        if cat_candidates:
+            counts = self.df_spark_.agg(
+                *[F.approx_count_distinct(c).alias(c) for c in cat_candidates]
+            ).first().asDict()
+            for c in cat_candidates:
+                if counts.get(c, 0) <= self.max_cat_levels or c in mandatory_set:
+                    keep_cat.append(c)
+                else:
+                    high_cardinality.append(c)
+
+        self.feature_types_ = {
+            "numeric": numeric_cols,
+            "categorical": keep_cat,
+            "datetime": datetime_cols,
+            "high_cardinality": high_cardinality,
+        }
+        self._log(
+            f"  Numeric: {len(numeric_cols)} | Categorical: {len(keep_cat)} | "
+            f"Datetime(excl): {len(datetime_cols)} | High-card(excl): {len(high_cardinality)}"
+        )
+        return self
+
+    def step_downcast(self):
+        if not self.downcast_enabled:
+            self._log("\n[Step 2/5] Skipping downcast.")
+            return self
+        self._log("\n[Step 2/5] Downcasting to 32-bit...")
+        protected = set([self.target_col] + self.mandatory_features)
+        cast_map = []
+        for field in self.df_spark_.schema.fields:
+            name, dtype = field.name, field.dataType
+            if name in protected:
+                cast_map.append(
+                    F.col(name).cast(DoubleType()).alias(name)
+                    if isinstance(dtype, DecimalType)
+                    else F.col(name)
+                )
+            elif isinstance(dtype, (DecimalType, DoubleType)):
+                cast_map.append(F.col(name).cast(FloatType()).alias(name))
+            else:
+                cast_map.append(F.col(name))
+        self.df_spark_ = self.df_spark_.select(*cast_map)
+        return self
+
+    def step_compute_iv(self):
+        if self.feature_types_ is None:
+            raise RuntimeError("Run step_identify_features() first.")
+        self._log(f"\n[Step 3/5] Computing Information Value ({self.iv_n_bins}-bin)...")
+        t0 = time.time()
+
+        numeric_cols = self.feature_types_["numeric"]
+        categorical_cols = self.feature_types_["categorical"]
+        select_cols = numeric_cols + categorical_cols + [self.target_col]
+        df_work = self.df_spark_.select(*select_cols).persist(StorageLevel.MEMORY_AND_DISK)
+        self._n_rows = df_work.count()
+        n_feat = len(numeric_cols) + len(categorical_cols)
+        self._log(f"  Rows: {self._n_rows:,} | Features: {n_feat}")
+
+        self._direct_ok = (self._n_rows <= 500_000 and n_feat <= 200)
+        if self._direct_ok:
+            self._log(f"  Small data - skipping IV, keeping ALL {n_feat} features.")
+            self.iv_results_ = pd.DataFrame({"feature": numeric_cols + categorical_cols, "iv": [None] * n_feat})
+            self._df_work = df_work
+            return self
+
+        iv_sdf = df_work.limit(self.iv_sample_rows) if self._n_rows > self.iv_sample_rows else df_work
+        iv_sdf = iv_sdf.filter(iv_sdf[self.target_col].isNotNull())
+        target_col = self.target_col
+        n_bins = self.iv_n_bins
+
+        totals = iv_sdf.agg(
+            F.count("*").alias("n"),
+            F.sum(F.col(target_col).cast("int")).alias("ev")
+        ).collect()[0]
+        n_total, total_ev = totals["n"], int(totals["ev"])
+        total_non_ev = n_total - total_ev
+        if total_ev == 0 or total_non_ev == 0:
+            raise ValueError("Target has only one class.")
+
+        results = []
+        q_probs = np.array([i / n_bins for i in range(1, n_bins)])
+
+        if numeric_cols:
+            hist_schema = StructType([
+                StructField("feature", StringType(), False),
+                StructField("bin_id", IntegerType(), False),
+                StructField("cnt", LongType(), False),
+                StructField("events", LongType(), False),
+            ])
+            n_cols_sel = len(numeric_cols) + 1
+            hist_parts = max(200, int(np.ceil(n_total * n_cols_sel * 4 / 25e6)))
+
+            sample_pdf = iv_sdf.select(*[F.col(c) for c in numeric_cols]).limit(50_000).toPandas()
+            bounds_map = {}
+            for c in numeric_cols:
+                vals = pd.to_numeric(sample_pdf[c], errors="coerce").dropna().to_numpy(dtype=np.float64)
+                if len(vals) < 10:
+                    continue
+                bds = np.unique(np.quantile(vals, q_probs))
+                if len(bds) >= 1:
+                    bounds_map[c] = bds
+            del sample_pdf
+
+            if bounds_map:
+                bounds_all = tuple(bounds_map.items())
+                empty_pdf = pd.DataFrame({
+                    "feature": pd.Series(dtype="str"),
+                    "bin_id": pd.Series(dtype="int32"),
+                    "cnt": pd.Series(dtype="int64"),
+                    "events": pd.Series(dtype="int64")
+                })
+
+                def _hist(pdf_iter, _b=bounds_all, _tc=target_col):
+                    for pdf in pdf_iter:
+                        tgt = pd.to_numeric(pdf[_tc], errors="coerce").fillna(0).astype(np.int64).to_numpy()
+                        rows = []
+                        for feat, bounds in _b:
+                            if feat not in pdf.columns:
+                                continue
+                            vals = pd.to_numeric(pdf[feat], errors="coerce").to_numpy(dtype=np.float64, copy=False)
+                            bins = np.searchsorted(bounds, np.nan_to_num(vals, nan=-np.inf), side="right").astype(np.int32)
+                            cnt = np.bincount(bins, minlength=len(bounds) + 1)
+                            ev = np.bincount(bins, weights=tgt, minlength=len(bounds) + 1)
+                            for b in np.nonzero(cnt)[0].tolist():
+                                rows.append((feat, int(b), int(cnt[b]), int(ev[b])))
+                        yield pd.DataFrame(rows, columns=["feature", "bin_id", "cnt", "events"]) if rows else empty_pdf
+
+                agg_rows = (
+                    iv_sdf.repartition(hist_parts)
+                    .select([F.col(target_col).cast("int").alias(target_col)] + [F.col(c) for c in bounds_map])
+                    .mapInPandas(_hist, schema=hist_schema)
+                    .groupBy("feature", "bin_id")
+                    .agg(F.sum("cnt").alias("cnt"), F.sum("events").alias("events"))
+                    .collect()
+                )
+
+                feat_bins = defaultdict(list)
+                for row in agg_rows:
+                    feat_bins[row["feature"]].append(row)
+                for feat, bins in feat_bins.items():
+                    if len(bins) < 2:
+                        continue
+                    iv = 0.0
+                    for row in bins:
+                        ev_i, non_ev_i = int(row["events"]), int(row["cnt"]) - int(row["events"])
+                        pct_ev = max(ev_i, 0.5) / total_ev
+                        pct_non = max(non_ev_i, 0.5) / total_non_ev
+                        iv += (pct_non - pct_ev) * np.log(pct_non / pct_ev)
+                    results.append({"feature": feat, "iv": iv})
+
+        for feat in categorical_cols:
+            rows = iv_sdf.filter(F.col(feat).isNotNull()).groupBy(feat).agg(
+                F.count("*").alias("cnt"),
+                F.sum(F.col(target_col).cast("int")).alias("events")
+            ).collect()
+            if len(rows) < 2:
+                continue
+            iv = 0.0
+            for row in rows:
+                ev_i, non_ev_i = int(row["events"]), int(row["cnt"]) - int(row["events"])
+                pct_ev = max(ev_i, 0.5) / total_ev
+                pct_non = max(non_ev_i, 0.5) / total_non_ev
+                iv += (pct_non - pct_ev) * np.log(pct_non / pct_ev)
+            results.append({"feature": feat, "iv": iv})
+
+        self.iv_results_ = pd.DataFrame(results).sort_values("iv", ascending=False).reset_index(drop=True) if results else pd.DataFrame(columns=["feature", "iv"])
+        self._df_work = df_work
+        n_pass = len(self.iv_results_[self.iv_results_["iv"] >= self.iv_threshold]) if len(self.iv_results_) > 0 else 0
+        self._log(f"  IV done in {time.time() - t0:.1f}s | Features with IV >= {self.iv_threshold}: {n_pass}")
+        return self
+
+    def step_prune_correlations(self):
+        if self.iv_results_ is None:
+            self._log("\n[Step 4/5] Skipping correlation pruning - no IV results. Run step_compute_iv() first.")
+            self.correlation_drops_ = []
+            self.final_features_ = sorted(set(self.mandatory_features) - {""})
+            return self
+        self._log(f"\n[Step 4/5] Correlation pruning (threshold={self.corr_threshold})...")
+        t0 = time.time()
+
+        if self._direct_ok:
+            self._log("  Skipped (small data).")
+            all_feats = self.feature_types_["numeric"] + self.feature_types_["categorical"]
+            self.correlation_drops_ = []
+            self.final_features_ = sorted(set(all_feats + self.mandatory_features) - {""})
+            return self
+
+        numeric_set = set(self.feature_types_["numeric"])
+        iv_df = self.iv_results_
+        iv_passing = iv_df[iv_df["iv"] >= self.iv_threshold]["feature"].tolist()
+        top_numeric = [f for f in iv_passing if f in numeric_set and f not in self.mandatory_features]
+        top_categoric = [f for f in iv_passing if f not in numeric_set and f not in self.mandatory_features]
+
+        iv_order = iv_df.set_index("feature")
+        ordered_numeric = iv_order.loc[[f for f in top_numeric if f in iv_order.index]].sort_values("iv", ascending=False).index.tolist()
+        if self.max_corr_features and len(ordered_numeric) > self.max_corr_features:
+            ordered_numeric = ordered_numeric[:self.max_corr_features]
+
+        kept_numeric, dropped_info = [], []
+        if ordered_numeric:
+            corr_sdf = (self._df_work.limit(self.corr_sample_rows) if self._n_rows > self.corr_sample_rows else self._df_work)
+            sdf = corr_sdf.select(*ordered_numeric).fillna(0)
+            vec = VectorAssembler(inputCols=ordered_numeric, outputCol="features", handleInvalid="keep").transform(sdf).select("features")
+            corr_mat = Correlation.corr(vec, "features", "pearson").head()[0]
+
+            kept_idx = []
+            for i, f in enumerate(ordered_numeric):
+                drop = None
+                for j in kept_idx:
+                    if abs(corr_mat[i, j]) >= self.corr_threshold:
+                        drop = (ordered_numeric[j], abs(corr_mat[i, j]))
+                        break
+                if drop:
+                    dropped_info.append({"feature": f, "correlated_with": drop[0], "correlation_value": round(float(drop[1]), 4)})
+                else:
+                    kept_numeric.append(f)
+                    kept_idx.append(i)
+
+            if len(kept_numeric) < self.expected_final_min and self.corr_threshold < 0.98:
+                th2 = min(0.99, self.corr_threshold + 0.02)
+                self._log(f"  Retrying with threshold={th2:.2f}")
+                kept_numeric, dropped_info, kept_idx = [], [], []
+                for i, f in enumerate(ordered_numeric):
+                    drop = None
+                    for j in kept_idx:
+                        if abs(corr_mat[i, j]) >= th2:
+                            drop = (ordered_numeric[j], abs(corr_mat[i, j]))
+                            break
+                    if drop:
+                        dropped_info.append({"feature": f, "correlated_with": drop[0], "correlation_value": round(float(drop[1]), 4)})
+                    else:
+                        kept_numeric.append(f)
+                        kept_idx.append(i)
+
+            if len(kept_numeric) > self.expected_final_max:
+                kept_numeric = kept_numeric[:self.expected_final_max]
+
+        self.correlation_drops_ = dropped_info
+        final_set = set(kept_numeric + top_categoric + self.mandatory_features)
+        final_set.discard("")
+        self.final_features_ = sorted(final_set)
+        self._log(f"  Done in {time.time() - t0:.1f}s | Kept: {len(kept_numeric)} numeric + {len(top_categoric)} cat | Dropped: {len(dropped_info)}")
+        self._log(f"  Final features: {len(self.final_features_)}")
+        return self
+
+    def step_build_report(self):
+        if self.final_features_ is None:
+            raise RuntimeError("Run step_prune_correlations() first.")
+        self._log("\n[Step 5/5] Building report & selecting columns...")
+
+        numeric_set = set(self.feature_types_["numeric"])
+        iv_lookup = dict(zip(self.iv_results_["feature"], self.iv_results_["iv"])) if len(self.iv_results_) > 0 else {}
+        corr_lookup = {d["feature"]: d for d in (self.correlation_drops_ or [])}
+        final_set = set(self.final_features_)
+        rows = []
+
+        for c in self.feature_types_["datetime"]:
+            rows.append({
+                "feature": c,
+                "dtype": "datetime",
+                "iv": None,
+                "correlated_with": None,
+                "correlation_value": None,
+                "is_selected": False,
+                "exclusion_reason": "datetime_column"
+            })
+        for c in self.feature_types_["high_cardinality"]:
+            rows.append({
+                "feature": c,
+                "dtype": "categorical",
+                "iv": None,
+                "correlated_with": None,
+                "correlation_value": None,
+                "is_selected": False,
+                "exclusion_reason": f"high_cardinality (>{self.max_cat_levels})"
+            })
+        for c in self.exclude_features:
+            if c not in [r["feature"] for r in rows]:
+                rows.append({
+                    "feature": c,
+                    "dtype": "excluded",
+                    "iv": None,
+                    "correlated_with": None,
+                    "correlation_value": None,
+                    "is_selected": False,
+                    "exclusion_reason": "user_excluded"
+                })
+
+        for c in self.feature_types_["numeric"] + self.feature_types_["categorical"]:
+            dtype_str = "numeric" if c in numeric_set else "categorical"
+            iv_val = iv_lookup.get(c)
+            corr_with, corr_val, is_sel = None, None, c in final_set
+            if c in self.mandatory_features:
+                reason = ""
+                is_sel = True
+            elif c in corr_lookup:
+                reason = f"correlated (r={corr_lookup[c]['correlation_value']:.3f})"
+                corr_with, corr_val = corr_lookup[c]["correlated_with"], corr_lookup[c]["correlation_value"]
+            elif iv_val is not None and iv_val < self.iv_threshold:
+                reason = f"low_iv ({iv_val:.4f})"
+            elif not is_sel:
+                reason = "not_selected"
+            else:
+                reason = ""
+            rows.append({
+                "feature": c,
+                "dtype": dtype_str,
+                "iv": round(float(iv_val), 6) if iv_val is not None else None,
+                "correlated_with": corr_with,
+                "correlation_value": corr_val,
+                "is_selected": is_sel,
+                "exclusion_reason": reason
+            })
+
+        self.feature_report_ = pd.DataFrame(rows).sort_values(
+            ["is_selected", "iv"], ascending=[False, False]
+        ).reset_index(drop=True)
+
+        existing = set(self._df_work.columns)
+        final_select = [c for c in self.final_features_ if c in existing] + [self.target_col]
+        self.result_df_ = self._df_work.select(*final_select)
+        try:
+            self._df_work.unpersist()
+        except Exception:
+            pass
+        return self
+
+    def run(self) -> Tuple[SparkDataFrame, pd.DataFrame]:
+        t0 = time.time()
+        self._log(f"\n{'='*60}\nFEATURE SELECTION PIPELINE\n{'='*60}")
+        self.step_identify_features()
+        self.step_downcast()
+        self.step_compute_iv()
+        self.step_prune_correlations()
+        self.step_build_report()
+        self._log(f"\n{'='*60}\nDONE in {time.time() - t0:.1f}s | {len(self.final_features_)} features\n{'='*60}\n")
+        return self.result_df_, self.feature_report_
+
+    def get_selected_columns(self) -> List[str]:
+        return self.feature_report_[self.feature_report_["is_selected"]]["feature"].tolist()
+
+    def get_iv_summary(self) -> pd.DataFrame:
+        return self.iv_results_[self.iv_results_["iv"].notna()].sort_values("iv", ascending=False).reset_index(drop=True)
+
+    def get_correlation_drops(self) -> pd.DataFrame:
+        if not self.correlation_drops_:
+            return pd.DataFrame(columns=["feature", "correlated_with", "correlation_value"])
+        return pd.DataFrame(self.correlation_drops_).sort_values("correlation_value", ascending=False).reset_index(drop=True)

feature_pruning-0.1.0.dist-info/METADATA

@@ -0,0 +1,330 @@
+Metadata-Version: 2.4
+Name: feature_pruning
+Version: 0.1.0
+Summary: High-performance PySpark library for scalable feature reduction and Information Value (IV) pruning in Credit Risk Scorecards
+Author-email: Vrukshya <vrukshyaai@gmail.com>
+Maintainer-email: Vrukshya <vrukshyaai@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2026 Vrukshya Org
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+Project-URL: Homepage, https://github.com/vrukshya/feature_pruning
+Project-URL: Documentation, https://github.com/vrukshya/feature_pruning#readme
+Project-URL: Repository, https://github.com/vrukshya/feature_pruning
+Project-URL: Bug Tracker, https://github.com/vrukshya/feature_pruning/issues
+Keywords: credit-risk,credit-scoring,scorecard,feature-reduction,feature-pruning,feature-selection,information-value,woe,pyspark,fintech,machine-learning
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Financial and Insurance Industry
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Office/Business :: Financial
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: numpy>=1.20.0
+Requires-Dist: pandas>=1.3.0
+Requires-Dist: pyspark>=3.1.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: flake8>=6.0.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Dynamic: license-file
+
+# feature_pruning
+
+[![Python Version](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue.svg)](https://pypi.org/project/feature-pruning/)
+[![PySpark](https://img.shields.io/badge/PySpark-%3E%3D3.1.0-orange.svg)](https://spark.apache.org/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+
+**High-Performance Distributed Feature Reduction Engine for Credit Risk Scorecards.**
+
+`feature_pruning` is an enterprise-grade Python library built natively on Apache Spark for selecting and pruning variables in credit risk scorecard development (Probability of Default / Basel II/III / IFRS 9 / Retail Scorecards).
+
+It automates the transition from thousands of raw credit bureau, transaction, and demographic attributes down to an optimal, highly predictive, non-collinear feature set ready for Weight of Evidence (WoE) binning and Logistic Regression.
+
+---
+
+## The Credit Risk Scorecard Challenge
+
+Building regulatory-compliant credit scorecards presents unique data engineering and modeling hurdles:
+
+- **Audit & Governance (SR 11-7 / Basel / IFRS 9)**: Model risk management (MRM) and regulatory auditors require an explicit justification for every discarded or retained variable.
+- **Extreme Multicollinearity**: Credit bureau tables often contain dozens of collinear metrics (e.g., `num_inquiries_3m`, `num_inquiries_6m`, `num_inquiries_12m`). In standard logistic regression scorecards, collinearity causes unstable coefficients and counter-intuitive sign reversals.
+- **Predictive Quality**: Features must meet minimum **Information Value (IV)** standards while preserving mandatory business or regulatory key indicators.
+- **Big Data Scale**: Modern credit datasets often span millions of accounts and thousands of features. Single-machine libraries (pandas/scikit-learn) crash with `OutOfMemory` errors when computing quantile cuts and pairwise correlations.
+
+`feature_pruning` solves these challenges by running distributed quantile binning, IV evaluation, and correlation pruning entirely within **Apache Spark**.
+
+---
+
+## Pipeline Architecture
+
+```
+                           Raw Spark DataFrame (Millions of Rows, 1000s of Features)
+                                                   │
+                                                   ▼
+┌────────────────────────────────────────────────────────────────────────────────────────────────────────┐
+│ Stage 1: Feature Classification                                                                        │
+│ • Detects Numeric vs. Categorical vs. Datetime columns                                                 │
+│ • Flags & filters high-cardinality strings (> max_cat_levels)                                          │
+│ • Protects mandatory features and excludes requested columns                                          │
+└──────────────────────────────────────────────────┬─────────────────────────────────────────────────────┘
+                                                   │
+                                                   ▼
+┌────────────────────────────────────────────────────────────────────────────────────────────────────────┐
+│ Stage 2: Distributed Precision Downcasting                                                             │
+│ • Downcasts DoubleType & DecimalType to 32-bit FloatType                                              │
+│ • Cuts executor memory consumption by ~50% during matrix aggregation                                  │
+└──────────────────────────────────────────────────┬─────────────────────────────────────────────────────┘
+                                                   │
+                                                   ▼
+┌────────────────────────────────────────────────────────────────────────────────────────────────────────┐
+│ Stage 3: High-Throughput Information Value (IV) Computation                                            │
+│ • Computes continuous quantiles and distributed binning via Spark mapInPandas                         │
+│ • Aggregates Goods/Bads and calculates IV for numeric and categorical variables                        │
+│ • Filters out variables with IV < iv_threshold                                                         │
+└──────────────────────────────────────────────────┬─────────────────────────────────────────────────────┘
+                                                   │
+                                                   ▼
+┌────────────────────────────────────────────────────────────────────────────────────────────────────────┐
+│ Stage 4: IV-Prioritized Correlation Pruning                                                            │
+│ • Computes Pearson correlation matrix via PySpark VectorAssembler + Correlation                        │
+│ • Between collinear pairs (r >= corr_threshold), retains the feature with higher IV                    │
+│ • Dynamic threshold relaxation ensures minimum required feature count is met                          │
+└──────────────────────────────────────────────────┬─────────────────────────────────────────────────────┘
+                                                   │
+                                                   ▼
+┌────────────────────────────────────────────────────────────────────────────────────────────────────────┐
+│ Stage 5: Full Audit Reporting & Data Delivery                                                          │
+│ • Generates comprehensive audit DataFrame (is_selected, iv, correlated_with, exclusion_reason)         │
+│ • Outputs pruned Spark DataFrame ready for Weight of Evidence (WoE) binning                           │
+└────────────────────────────────────────────────────────────────────────────────────────────────────────┘
+```
+
+---
+
+## Key Features
+
+- ⚡ **Native PySpark Scalability**: Distributed quantile histograms and correlation matrices executed directly on cluster workers via Spark ML and `mapInPandas`.
+- 📊 **IV-Driven Pruning**: When two features are collinear, the pipeline drops the weaker predictor and retains the variable with higher Information Value.
+- 🛡️ **Mandatory Variable Protection**: Ensure business-critical variables (e.g., debt-to-income, credit bureau score) are never removed, regardless of their statistical properties.
+- 📋 **Regulatory Audit Trail**: Automatically produces a full governance table explaining why every single column was accepted or eliminated (e.g., `low_iv (0.012)`, `correlated (r=0.962)`, `high_cardinality (>200)`, or `datetime_column`).
+- 🔄 **Dynamic Threshold Relaxation**: Automatically adjusts correlation thresholds if filtering becomes overly aggressive, keeping feature counts within target bounds (`expected_final_min`, `expected_final_max`).
+
+---
+
+## Installation
+
+### From PyPI
+```bash
+pip install feature-pruning
+```
+
+### For Local Development
+```bash
+git clone https://github.com/vrukshya/feature_pruning.git
+cd feature_pruning
+pip install -e ".[dev]"
+```
+
+### In Databricks / Cloud Notebooks
+In your Databricks notebook cell:
+```python
+%pip install feature-pruning
+```
+Or add `feature-pruning` to your Databricks cluster libraries.
+
+---
+
+## Quickstart
+
+```python
+from pyspark.sql import SparkSession
+from feature_pruning import FeatureSelectionPipeline
+
+# 1. Initialize Spark session (or use active session in Databricks/EMR)
+spark = SparkSession.builder.appName("CreditRiskScorecard").getOrCreate()
+
+# 2. Load credit training data
+df = spark.table("risk_catalog.credit_data.application_train")
+
+# 3. Configure the feature selection pipeline
+pipeline = FeatureSelectionPipeline(
+    df=df,
+    target_col="default_flag",              # 0 = Good loan, 1 = Default / Bad loan
+    mandatory_features=["bureau_score", "dti_ratio"],  # Keep regardless of correlation
+    exclude_features=["application_id", "ssn_hash"],   # Exclude identifiers
+    iv_threshold=0.03,                      # Industry baseline: IV >= 0.03
+    corr_threshold=0.95,                    # Multicollinearity cutoff
+    expected_final_min=20,
+    expected_final_max=150,
+    verbose=True,
+)
+
+# 4. Execute pipeline
+selected_spark_df, audit_report = pipeline.run()
+
+# 5. Review results
+print("Selected features count:", len(pipeline.get_selected_columns()))
+print("Selected columns:", pipeline.get_selected_columns())
+
+# 6. Inspect audit report
+print(audit_report.head(20))
+```
+
+---
+
+## Inspection & Diagnostic Methods
+
+After calling `pipeline.run()`, several diagnostic helpers allow inspection of the feature reduction decisions:
+
+```python
+# 1. Retrieve list of final selected feature names
+selected_features = pipeline.get_selected_columns()
+
+# 2. Inspect Information Value ranking for all evaluated features
+iv_summary = pipeline.get_iv_summary()
+print(iv_summary.head(10))
+
+# 3. Inspect which features were pruned due to correlation and their collinear counterpart
+correlation_drops = pipeline.get_correlation_drops()
+print(correlation_drops.head(10))
+
+# 4. Full audit report with exact exclusion reasons
+print(pipeline.feature_report_)
+```
+
+### Sample Audit Report Output
+
+| feature | dtype | iv | correlated_with | correlation_value | is_selected | exclusion_reason |
+|---|---|---|---|---|---|---|
+| `bureau_score` | numeric | 0.4521 | None | None | True | |
+| `utilization_rate`| numeric | 0.3180 | None | None | True | |
+| `num_inquiries_6m` | numeric | 0.1420 | None | None | True | |
+| `num_inquiries_3m` | numeric | 0.1210 | `num_inquiries_6m`| 0.965 | False | correlated (r=0.965) |
+| `employer_name` | categorical | None | None | None | False | high_cardinality (>200) |
+| `postal_code_raw`| numeric | 0.0120 | None | None | False | low_iv (0.0120) |
+| `application_date`| datetime | None | None | None | False | datetime_column |
+
+---
+
+## API Reference
+
+### `FeatureSelectionPipeline`
+
+```python
+FeatureSelectionPipeline(
+    df: SparkDataFrame,
+    target_col: str,
+    mandatory_features: Optional[List[str]] = None,
+    exclude_features: Optional[List[str]] = None,
+    iv_threshold: float = 0.03,
+    iv_n_bins: int = 100,
+    iv_sample_rows: int = 1_000_000,
+    corr_threshold: float = 0.95,
+    corr_sample_rows: int = 500_000,
+    max_corr_features: int = 5000,
+    max_cat_levels: int = 200,
+    expected_final_min: int = 200,
+    expected_final_max: int = 1000,
+    downcast: bool = True,
+    verbose: bool = True,
+)
+```
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `df` | `SparkDataFrame` | *Required* | Input Spark DataFrame containing raw candidate variables and target. |
+| `target_col` | `str` | *Required* | Name of the binary target (0 = Good, 1 = Bad). Case-insensitive. |
+| `mandatory_features` | `List[str]` | `None` | Columns guaranteed to be retained regardless of IV or correlation. |
+| `exclude_features` | `List[str]` | `None` | Columns explicitly excluded from candidate pool (IDs, timestamps). |
+| `iv_threshold` | `float` | `0.03` | Minimum Information Value (IV) required for feature inclusion. |
+| `iv_n_bins` | `int` | `100` | Quantile bin resolution for continuous variable histogram computation. |
+| `iv_sample_rows` | `int` | `1,000,000` | Sample ceiling for computing quantile thresholds on continuous columns. |
+| `corr_threshold` | `float` | `0.95` | Pearson correlation ceiling. Collinear variable with lower IV is pruned. |
+| `corr_sample_rows` | `int` | `500,000` | Sample ceiling for Pearson correlation matrix calculation. |
+| `max_corr_features` | `int` | `5000` | Upper limit of top IV features fed into correlation assembler. |
+| `max_cat_levels` | `int` | `200` | Maximum unique levels allowed before categorical is dropped as high-cardinality. |
+| `expected_final_min` | `int` | `200` | Minimum retained count. Relaxes `corr_threshold` if pruned too aggressively. |
+| `expected_final_max` | `int` | `1000` | Maximum cap on final retained variables. |
+| `downcast` | `bool` | `True` | Downcasts Decimal/Double columns to Float32 to optimize Spark worker memory. |
+| `verbose` | `bool` | `True` | Prints stage progress and execution timing logs. |
+
+---
+
+## Credit Risk Scorecard Rules of Thumb
+
+### Information Value (IV) Benchmarks
+
+In credit risk modeling (Siddiqi, 2005), Information Value serves as the primary metric for filtering out uninformative signals:
+
+| Information Value (IV) | Predictive Power | Action in Scorecard Development |
+|---|---|---|
+| **< 0.02** | Unpredictive | **Drop**: Adds noise and degrees of freedom without signal. |
+| **0.02 – 0.10** | Weak Predictor | **Evaluate**: May be retained if part of a key credit policy dimension. |
+| **0.10 – 0.30** | Medium Predictor | **Keep**: Core candidate for scorecard inclusion. |
+| **0.30 – 0.50** | Strong Predictor | **Keep**: High diagnostic quality feature. |
+| **> 0.50** | Suspicious / Too Good | **Investigate**: Often indicative of target leakage or operational bias. |
+
+### Correlation Thresholds
+
+- Standard practice sets the correlation threshold between `0.80` and `0.95`.
+- Setting `corr_threshold=0.95` catches near-duplicate metrics (e.g., balance in dollars vs. balance in thousands).
+- Setting `corr_threshold=0.85` produces a tighter, more orthogonal set of features that prevents variance inflation in final logistic regression models.
+
+---
+
+## Performance & Spark Optimization
+
+- **Quantile Binning (`mapInPandas`)**: Rather than running expensive full-dataset sorting on each column, `feature_pruning` samples continuous variables to establish robust quantile boundaries, then computes distributed frequency histograms across partitions in a single pass.
+- **Spark Storage Management**: Intermediate working DataFrames are cached at `StorageLevel.MEMORY_AND_DISK` and unpersisted automatically at the conclusion of report generation.
+- **Vector Correlation**: Correlation is calculated using Spark ML's native distributed linear algebra (`Correlation.corr`), supporting thousands of features simultaneously.
+
+---
+
+## Contributing
+
+Contributions, bug reports, and feature requests are welcome!
+Please feel free to submit a pull request or open an issue on GitHub.
+
+1. Fork the Project
+2. Create your Feature Branch (`git checkout -b feature/AmazingFeature`)
+3. Commit your Changes (`git commit -m 'Add some AmazingFeature'`)
+4. Push to the Branch (`git push origin feature/AmazingFeature`)
+5. Open a Pull Request
+
+---
+
+## License
+
+Distributed under the MIT License. See [LICENSE.txt](LICENSE.txt) for more details.

feature_pruning-0.1.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+feature_pruning/__init__.py,sha256=u3GiIQIRMSJVV-Q2mHSx193C68XJBwPViioxwFHlqlY,1407
+feature_pruning/core.py,sha256=SbTNSoyBvXV0y78_fN74rYntD7XxgpD-3yjqd_4k8FM,23851
+feature_pruning-0.1.0.dist-info/licenses/LICENSE.txt,sha256=Ifzkhdu7jmu8cJ7gJ9kb3HPfz2ZTSwJ6B6BUhJAXjm4,1088
+feature_pruning-0.1.0.dist-info/METADATA,sha256=NzEQJbUyS6E8rgDRSHYJy2PRglu_hCD1Uy1L0hfaDqU,20280
+feature_pruning-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+feature_pruning-0.1.0.dist-info/top_level.txt,sha256=mC52rgCaN2tZ8Nmrgb9cfNYSRWXfzFPC_CrWcmoQQ_w,16
+feature_pruning-0.1.0.dist-info/RECORD,,

feature_pruning-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

feature_pruning-0.1.0.dist-info/licenses/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Vrukshya Org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

feature_pruning-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+feature_pruning