frameprep 0.1.0__py3-none-any.whl

frameprep/__init__.py

@@ -0,0 +1,27 @@
+"""
+frameprep — Automated ML-Ready Data Preparation Library
+=======================================================
+A single pipeline object that ingests a raw DataFrame and outputs a fully
+ML-ready dataset: dtype inference, missing value imputation, encoding,
+scaling, leakage detection, and audit reporting.
+"""
+
+from frameprep.pipeline import frameprepPipeline
+from frameprep.core.dtype_inferrer import DTypeInferrer
+from frameprep.core.missing_strategy import MissingValueHandler
+from frameprep.encoders.categorical import CategoricalEncoder
+from frameprep.scalers.numeric import NumericScaler
+from frameprep.detectors.leakage import LeakageDetector
+from frameprep.reports.audit import AuditReport
+
+__version__ = "0.1.0"
+__author__ = "frameprep contributors"
+__all__ = [
+    "frameprepPipeline",
+    "DTypeInferrer",
+    "MissingValueHandler",
+    "CategoricalEncoder",
+    "NumericScaler",
+    "LeakageDetector",
+    "AuditReport",
+]

frameprep/core/__init__.py

@@ -0,0 +1,4 @@
+from frameprep.core.dtype_inferrer import DTypeInferrer
+from frameprep.core.missing_strategy import MissingValueHandler
+
+__all__ = ["DTypeInferrer", "MissingValueHandler"]

frameprep/core/dtype_inferrer.py

@@ -0,0 +1,238 @@
+"""
+Module 1 — Smart dtype inference + memory optimisation
+-------------------------------------------------------
+- Detects true column types from raw data (int stored as float, bool stored
+  as object, datetime strings, IDs, etc.)
+- Downcasts numerics to the smallest type that fits the data
+- Converts object columns that are actually categorical to pd.Categorical
+- Records every change in self.report_
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Dict, Any, cast
+
+import numpy as np
+import pandas as pd
+
+from frameprep.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+# Patterns for datetime detection
+_DATETIME_PATTERNS = [
+    r"\d{4}-\d{2}-\d{2}",            # 2023-01-15
+    r"\d{2}/\d{2}/\d{4}",            # 15/01/2023
+    r"\d{4}/\d{2}/\d{2}",            # 2023/01/15
+    r"\d{2}-\w{3}-\d{4}",            # 15-Jan-2023
+]
+_DT_RE = re.compile("|".join(_DATETIME_PATTERNS))
+
+# Fraction of rows that must match pattern to call it datetime
+_DATETIME_SAMPLE_THRESHOLD = 0.8
+# If unique values / total rows < this, treat as categorical
+_CARDINALITY_RATIO = 0.05
+
+
+class DTypeInferrer:
+    """
+    Infer and optimise dtypes for every column in a DataFrame.
+
+    After fit_transform, self.report_ contains:
+        {
+          "column_name": {
+              "original_dtype": ...,
+              "inferred_dtype": ...,
+              "memory_before_kb": ...,
+              "memory_after_kb": ...,
+          },
+          ...
+          "__summary__": {
+              "total_memory_before_kb": ...,
+              "total_memory_after_kb": ...,
+              "savings_pct": ...,
+          }
+        }
+    """
+
+    def __init__(self) -> None:
+        self.report_: Dict[str, Any] = {}
+        self._col_dtypes: Dict[str, str] = {}   # for transform()
+
+    # ------------------------------------------------------------------
+    # Public
+    # ------------------------------------------------------------------
+
+    def fit_transform(self, df: pd.DataFrame) -> pd.DataFrame:
+        df = df.copy()
+        mem_before = self._col_memory_kb(df)
+        col_reports: Dict[str, Any] = {}
+
+        for col in df.columns:
+            orig_dtype = str(df[col].dtype)
+            # Explicitly cast to pd.Series so the type checker knows the
+            # column accessor returns a Series, not Series | DataFrame.
+            series: pd.Series = cast(pd.Series, df[col])
+            df[col], new_dtype = self._infer_col(series)
+            self._col_dtypes[col] = new_dtype
+            col_reports[col] = {
+                "original_dtype": orig_dtype,
+                "inferred_dtype": new_dtype,
+            }
+
+        mem_after = self._col_memory_kb(df)
+        total_before = sum(mem_before.values())
+        total_after = sum(mem_after.values())
+
+        for col in col_reports:
+            col_reports[col]["memory_before_kb"] = round(mem_before.get(col, 0), 3)
+            col_reports[col]["memory_after_kb"] = round(mem_after.get(col, 0), 3)
+
+        savings = (
+            (total_before - total_after) / total_before * 100
+            if total_before > 0
+            else 0.0
+        )
+        col_reports["__summary__"] = {
+            "total_memory_before_kb": round(total_before, 3),
+            "total_memory_after_kb": round(total_after, 3),
+            "savings_pct": round(savings, 2),
+        }
+
+        self.report_ = col_reports
+        logger.debug(
+            f"Memory optimisation: {total_before:.1f} KB → "
+            f"{total_after:.1f} KB ({savings:.1f}% saved)"
+        )
+        return df
+
+    def transform(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Apply previously inferred dtypes to new data."""
+        df = df.copy()
+        for col, dtype in self._col_dtypes.items():
+            if col not in df.columns:
+                continue
+            try:
+                if dtype == "datetime64[ns]":
+                    df[col] = pd.to_datetime(df[col], errors="coerce")
+                elif dtype == "category":
+                    df[col] = df[col].astype("category")
+                elif dtype == "boolean":
+                    df[col] = df[col].astype("boolean")
+                else:
+                    df[col] = df[col].astype(dtype, errors="ignore")  # type: ignore[arg-type]
+            except Exception:
+                pass
+        return df
+
+    # ------------------------------------------------------------------
+    # Private
+    # ------------------------------------------------------------------
+
+    def _infer_col(self, series: pd.Series) -> tuple[pd.Series, str]:
+        """Return (transformed_series, dtype_string)."""
+
+        if pd.api.types.is_datetime64_any_dtype(series):
+            return series, "datetime64[ns]"
+
+        if pd.api.types.is_bool_dtype(series):
+            return series.astype("boolean"), "boolean"
+
+        if pd.api.types.is_float_dtype(series):
+            return self._try_downcast_float(series)
+
+        if pd.api.types.is_integer_dtype(series):
+            return self._try_downcast_int(series)
+
+        # Handle both legacy object dtype and pandas 2.x StringDtype
+        if pd.api.types.is_object_dtype(series) or isinstance(
+            series.dtype, pd.StringDtype
+        ):
+            return self._infer_object_col(series)
+
+        return series, str(series.dtype)
+
+    def _infer_object_col(
+        self, series: pd.Series
+    ) -> tuple[pd.Series, str]:
+        """Handle string/object columns."""
+        non_null = series.dropna()
+        if len(non_null) == 0:
+            return series, "object"
+
+        # Boolean strings
+        bool_map = {
+            "true": True, "false": False,
+            "yes": True, "no": False,
+            "1": True, "0": False,
+        }
+        lower_vals = non_null.astype(str).str.lower().unique()
+        if set(lower_vals).issubset(bool_map.keys()):
+            mapped = series.astype(str).str.lower().map(bool_map)
+            return mapped.astype("boolean"), "boolean"
+
+        # Datetime strings
+        sample = non_null.astype(str).head(200)
+        dt_matches = sample.apply(lambda v: bool(_DT_RE.search(v))).mean()
+        if dt_matches >= _DATETIME_SAMPLE_THRESHOLD:
+            converted = pd.to_datetime(series, errors="coerce")
+            if converted.notna().mean() >= _DATETIME_SAMPLE_THRESHOLD:
+                return converted, "datetime64[ns]"
+
+        # Numeric strings
+        # pd.to_numeric return type in the stubs is ambiguous (float | Series |
+        # DataFrame), so we cast explicitly to pd.Series to keep the type
+        # checker happy and get proper Series method access.
+        converted_num: pd.Series = pd.Series(
+            pd.to_numeric(series, errors="coerce"), index=series.index
+        )
+        if converted_num.notna().mean() >= 0.9:
+            if bool((converted_num.dropna() % 1 == 0).all()):
+                int_series: pd.Series = converted_num.astype("Int64")
+                return self._try_downcast_int(int_series)
+            return self._try_downcast_float(converted_num)
+
+        # Low-cardinality → categorical
+        ratio = series.nunique(dropna=True) / max(len(series), 1)
+        if ratio <= _CARDINALITY_RATIO:
+            return series.astype("category"), "category"
+
+        return series, "object"
+
+    def _try_downcast_float(
+        self, series: pd.Series
+    ) -> tuple[pd.Series, str]:
+        """Downcast float64 → float32 where safe."""
+        if pd.api.types.is_float_dtype(series):
+            s32: pd.Series = series.astype("float32")
+            # Tolerance check: max relative error < 1e-5
+            mask: pd.Series = series.notna()
+            if mask.any():
+                orig = np.asarray(series[mask], dtype="float64")
+                down = np.asarray(s32[mask], dtype="float64")
+                rel_err = np.abs(orig - down) / (np.abs(orig) + 1e-10)
+                if rel_err.max() < 1e-5:
+                    return s32, "float32"
+        return series, str(series.dtype)
+
+    def _try_downcast_int(
+        self, series: pd.Series
+    ) -> tuple[pd.Series, str]:
+        """Downcast int to smallest signed type that fits."""
+        try:
+            # pd.to_numeric downcast returns ambiguous type in stubs;
+            # wrap in pd.Series to narrow the type explicitly.
+            downcasted: pd.Series = pd.Series(
+                pd.to_numeric(series, downcast="integer"), index=series.index
+            )
+            return downcasted, str(downcasted.dtype)
+        except Exception:
+            return series, str(series.dtype)
+
+    @staticmethod
+    def _col_memory_kb(df: pd.DataFrame) -> Dict[str, float]:
+        return {
+            col: df[col].memory_usage(deep=True) / 1024
+            for col in df.columns
+        }

frameprep/core/missing_strategy.py

@@ -0,0 +1,260 @@
+"""
+Module 2 — Missing value strategy selector
+-------------------------------------------
+Uses statistical tests to pick the best imputation method per column:
+
+  Numeric columns
+  ---------------
+  • If missingness < 1 % → median fill (fast, safe)
+  • Run Little's MCAR test proxy (chi-square on missingness indicator vs
+    other columns). If MCAR → mean/median. If MAR → KNN imputer.
+  • If skewness |skew| > 1 → median, else → mean
+  • If high missingness (> 40 %) → flag + constant fill
+
+  Categorical / boolean columns
+  ------------------------------
+  • Mode fill if missingness < 30 %
+  • Otherwise add "__missing__" sentinel category
+
+Records every decision in self.report_.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, cast
+
+import numpy as np
+import pandas as pd
+from sklearn.impute import KNNImputer, SimpleImputer
+
+from frameprep.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+_HIGH_MISSING_THRESHOLD = 0.40
+_LOW_MISSING_THRESHOLD = 0.01
+_MAR_CORR_THRESHOLD = 0.10   # missingness indicator correlation
+
+
+class MissingValueHandler:
+    """
+    Automatically selects and applies the best imputation strategy
+    for each column.
+
+    After fit_transform, self.report_ contains per-column decisions:
+        {
+          "column_name": {
+              "missing_pct": 12.3,
+              "strategy": "knn",
+              "reason": "MAR detected via missingness correlation",
+          },
+          ...
+        }
+    """
+
+    def __init__(self, knn_neighbors: int = 5) -> None:
+        self.knn_neighbors = knn_neighbors
+        self.report_: Dict[str, Any] = {}
+        self._strategies: Dict[str, str] = {}
+        self._imputers: Dict[str, Any] = {}
+
+    # ------------------------------------------------------------------
+    # Public
+    # ------------------------------------------------------------------
+
+    def fit_transform(self, df: pd.DataFrame) -> pd.DataFrame:
+        df = df.copy()
+
+        num_cols = df.select_dtypes(include="number").columns.tolist()
+        cat_cols = df.select_dtypes(
+            exclude="number"
+        ).columns.tolist()
+
+        # Fit + transform numeric
+        df, num_reports = self._handle_numeric(df, num_cols, fit=True)
+
+        # Fit + transform categorical
+        df, cat_reports = self._handle_categorical(df, cat_cols, fit=True)
+
+        self.report_ = {**num_reports, **cat_reports}
+        self._log_summary()
+        return df
+
+    def transform(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Apply fitted imputers to new data."""
+        df = df.copy()
+        for col, imputer in self._imputers.items():
+            if col not in df.columns:
+                continue
+            strategy = self._strategies[col]
+            if strategy == "sentinel":
+                df[col] = df[col].fillna("__missing__")
+            elif strategy in ("mean", "median", "mode"):
+                df[[col]] = imputer.transform(df[[col]])
+            elif strategy == "knn":
+                df[[col]] = imputer.transform(df[[col]])
+            elif strategy == "constant_flag":
+                df[col] = df[col].fillna(-999)
+        return df
+
+    # ------------------------------------------------------------------
+    # Private — numeric
+    # ------------------------------------------------------------------
+
+    def _handle_numeric(
+        self,
+        df: pd.DataFrame,
+        cols: List[str],
+        fit: bool = True,
+    ) -> tuple[pd.DataFrame, Dict[str, Any]]:
+        reports: Dict[str, Any] = {}
+
+        # Identify columns that may need KNN (expensive, batch them)
+        knn_cols: List[str] = []
+
+        for col in cols:
+            missing_pct = df[col].isna().mean() * 100
+            strategy, reason = self._choose_numeric_strategy(
+                cast(pd.Series, df[col]), missing_pct, df, cols
+            )
+            reports[col] = {
+                "missing_pct": round(missing_pct, 2),
+                "strategy": strategy,
+                "reason": reason,
+            }
+            self._strategies[col] = strategy
+            if strategy == "knn":
+                knn_cols.append(col)
+
+        # Non-KNN imputers
+        for col in cols:
+            strategy = self._strategies[col]
+            if col in knn_cols:
+                continue
+            if strategy == "constant_flag":
+                df[col] = df[col].fillna(-999)
+                self._imputers[col] = None
+            elif strategy in ("mean", "median"):
+                imp = SimpleImputer(strategy=strategy)
+                if fit:
+                    df[[col]] = imp.fit_transform(df[[col]])
+                    self._imputers[col] = imp
+                else:
+                    df[[col]] = self._imputers[col].transform(df[[col]])
+            # zero-missing columns need no imputer
+            else:
+                self._imputers[col] = None
+
+        # KNN batch
+        if knn_cols:
+            knn = KNNImputer(n_neighbors=self.knn_neighbors)
+            if fit:
+                df[knn_cols] = knn.fit_transform(df[knn_cols])
+                for col in knn_cols:
+                    self._imputers[col] = knn
+            else:
+                df[knn_cols] = self._imputers[knn_cols[0]].transform(
+                    df[knn_cols]
+                )
+
+        return df, reports
+
+    def _choose_numeric_strategy(
+        self,
+        series: pd.Series,
+        missing_pct: float,
+        df: pd.DataFrame,
+        num_cols: List[str],
+    ) -> tuple[str, str]:
+        if missing_pct == 0:
+            return "none", "no missing values"
+
+        if missing_pct >= _HIGH_MISSING_THRESHOLD * 100:
+            return (
+                "constant_flag",
+                f"{missing_pct:.1f}% missing — too high for reliable imputation; flagged with -999",
+            )
+
+        if missing_pct < _LOW_MISSING_THRESHOLD * 100:
+            return "median", "< 1% missing — fast median fill"
+
+        # Check MAR: correlation between missingness indicator and other numerics
+        if self._is_mar(series, df, num_cols):
+            return "knn", "MAR detected via missingness correlation — KNN imputer"
+
+        skew = series.skew()
+        if abs(cast(float, skew)) > 1:
+            return "median", f"skewness={skew:.2f} > 1 — median is robust to outliers"
+        return "mean", f"skewness={skew:.2f} ≤ 1 — mean imputation"
+
+    @staticmethod
+    def _is_mar(
+        series: pd.Series,
+        df: pd.DataFrame,
+        num_cols: List[str],
+    ) -> bool:
+        """
+        Proxy for MAR: if the missingness indicator for this column
+        correlates with any other numeric column, likely MAR.
+        """
+        if series.isna().sum() == 0:
+            return False
+        indicator = series.isna().astype(int)
+        others = [c for c in num_cols if c != series.name]
+        for other in others:
+            try:
+                corr = abs(indicator.corr(cast(pd.Series, df[other].fillna(df[other].median()))))
+                if corr > _MAR_CORR_THRESHOLD:
+                    return True
+            except Exception:
+                continue
+        return False
+
+    # ------------------------------------------------------------------
+    # Private — categorical
+    # ------------------------------------------------------------------
+
+    def _handle_categorical(
+        self,
+        df: pd.DataFrame,
+        cols: List[str],
+        fit: bool = True,
+    ) -> tuple[pd.DataFrame, Dict[str, Any]]:
+        reports: Dict[str, Any] = {}
+        for col in cols:
+            missing_pct = df[col].isna().mean() * 100
+            if missing_pct == 0:
+                strategy, reason = "none", "no missing values"
+            elif missing_pct < 30:
+                strategy, reason = "mode", "< 30% missing — mode fill"
+            else:
+                strategy, reason = (
+                    "sentinel",
+                    f"{missing_pct:.1f}% missing — sentinel '__missing__' added",
+                )
+
+            reports[col] = {
+                "missing_pct": round(missing_pct, 2),
+                "strategy": strategy,
+                "reason": reason,
+            }
+            self._strategies[col] = strategy
+
+            if strategy == "mode":
+                if fit:
+                    imp = SimpleImputer(strategy="most_frequent")
+                    df[[col]] = imp.fit_transform(df[[col]])
+                    self._imputers[col] = imp
+                else:
+                    df[[col]] = self._imputers[col].transform(df[[col]])
+            elif strategy == "sentinel":
+                df[col] = df[col].fillna("__missing__")
+                self._imputers[col] = None
+
+        return df, reports
+
+    def _log_summary(self) -> None:
+        strategies = [v["strategy"] for v in self.report_.values()]
+        from collections import Counter
+        counts = Counter(strategies)
+        logger.debug(f"Missing value strategies applied: {dict(counts)}")

frameprep/detectors/__init__.py

@@ -0,0 +1,3 @@
+from frameprep.detectors.leakage import LeakageDetector
+
+__all__ = ["LeakageDetector"]