tanml 0.1.6__py3-none-any.whl → 0.1.7__py3-none-any.whl

tanml/checks/performance_classification.py

@@ -0,0 +1,305 @@
+from __future__ import annotations
+import os, math
+from dataclasses import dataclass, asdict
+from typing import Dict, Any
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.metrics import (
+    roc_auc_score, roc_curve, confusion_matrix, precision_recall_fscore_support,
+    accuracy_score, average_precision_score, brier_score_loss, precision_recall_curve
+)
+from sklearn.calibration import calibration_curve
+
+# ---------- utilities ----------
+def _ensure_dir(d: str) -> str:
+    os.makedirs(d, exist_ok=True)
+    return d
+
+def _savefig(path: str) -> str:
+    plt.tight_layout()
+    plt.savefig(path, bbox_inches="tight", dpi=160)
+    plt.close()
+    return path
+
+def _gini_from_auc(auc: float) -> float:
+    return 2 * auc - 1 if (auc is not None and not np.isnan(auc)) else np.nan
+
+def _ks_from_roc(fpr, tpr) -> float:
+    return float(np.max(np.abs(tpr - fpr))) if len(fpr) else np.nan
+
+def _decile_lift_table(y_true: np.ndarray, y_score: np.ndarray, pos_label: int = 1, n_bins: int = 10) -> pd.DataFrame:
+    df = pd.DataFrame({"y": (y_true == pos_label).astype(int), "score": y_score})
+    df = df.sort_values("score", ascending=False).reset_index(drop=True)
+    df["decile"] = pd.qcut(df.index + 1, q=n_bins, labels=list(range(1, n_bins + 1)))
+    g = df.groupby("decile", observed=True).agg(
+        total=("y", "size"),
+        events=("y", "sum"),
+        avg_score=("score", "mean"),
+    ).reset_index()
+    g["event_rate"] = g["events"] / g["total"]
+    overall_rate = df["y"].mean() if df["y"].size else np.nan
+    g["lift"] = g["event_rate"] / overall_rate if (overall_rate and not math.isclose(overall_rate, 0.0)) else np.nan
+
+    # cumulative capture & gain
+    g["cum_events"] = g["events"].cumsum()
+    g["cum_total"] = g["total"].cumsum()
+    total_events = g["events"].sum()
+    g["cum_capture_rate"] = g["cum_events"] / total_events if total_events > 0 else np.nan
+    g["cum_population"] = g["cum_total"] / g["total"].sum()
+    g["cum_gain"] = g["cum_capture_rate"]  # same as cumulative gains curve
+    return g
+
+def _ks_curve_frame(y_true: np.ndarray, y_score: np.ndarray, pos_label: int = 1) -> pd.DataFrame:
+    """
+    Returns a dataframe with columns:
+      population (fraction 0..1), cum_event, cum_non_event, ks_gap
+    sorted by score DESC, which is standard for risk ranking.
+    """
+    df = pd.DataFrame({"y": (y_true == pos_label).astype(int), "score": y_score})
+    if df.empty:
+        return pd.DataFrame(columns=["population", "cum_event", "cum_non_event", "ks_gap"])
+
+    df = df.sort_values("score", ascending=False).reset_index(drop=True)
+    n = len(df)
+
+    # counts
+    total_events = df["y"].sum()
+    total_non_events = n - total_events
+
+    # avoid divide-by-zero; if all one class, return empty (plotter will handle)
+    if total_events == 0 or total_non_events == 0:
+        return pd.DataFrame(columns=["population", "cum_event", "cum_non_event", "ks_gap"])
+
+    cum_events = np.cumsum(df["y"].values) / total_events
+    cum_non_events = np.cumsum(1 - df["y"].values) / total_non_events
+    population = (np.arange(1, n + 1)) / n
+    ks_gap = np.abs(cum_events - cum_non_events)
+
+    return pd.DataFrame({
+        "population": population,
+        "cum_event": cum_events,
+        "cum_non_event": cum_non_events,
+        "ks_gap": ks_gap
+    })
+
+@dataclass
+class ClassificationSummary:
+    auc: float
+    ks: float
+    accuracy: float
+    precision: float
+    recall: float
+    f1: float
+    pr_auc: float
+    brier: float
+    gini: float
+    # paths
+    roc_png: str
+    pr_png: str
+    lift_png: str
+    calib_png: str
+    cm_png: str
+    ks_png: str        
+    # tables
+    confusion_csv: str
+    lift_csv: str
+
+
+def compute_classification_report(
+    *,
+    y_true: np.ndarray,
+    y_score: np.ndarray,
+    y_pred: np.ndarray,
+    outdir: str,
+    pos_label: int = 1,
+    title_prefix: str = "Model"
+) -> Dict[str, Any]:
+    """
+    Computes metrics + saves plots/CSVs for classification.
+    Returns a dict ready for ReportBuilder/Jinja.
+    """
+    _ensure_dir(outdir)
+
+    # --- metrics
+    has_posneg = len(np.unique(y_true)) > 1
+    auc = roc_auc_score(y_true, y_score) if has_posneg else np.nan
+    fpr, tpr, _ = roc_curve(y_true, y_score) if has_posneg else (np.array([]), np.array([]), None)
+    ks = _ks_from_roc(fpr, tpr)
+    pr_auc = average_precision_score(y_true, y_score) if has_posneg else np.nan
+    brier = brier_score_loss(y_true, y_score)
+    gini = _gini_from_auc(auc)
+
+    precision, recall, f1, _ = precision_recall_fscore_support(
+        y_true, y_pred, average="binary", pos_label=pos_label, zero_division=0
+    )
+    acc = accuracy_score(y_true, y_pred)
+
+    # --- confusion matrix & CSV
+    cm = confusion_matrix(y_true, y_pred, labels=[0, 1])
+    cm_df = pd.DataFrame(cm, index=["Actual 0", "Actual 1"], columns=["Pred 0", "Pred 1"])
+    cm_csv = os.path.join(outdir, "confusion_matrix.csv")
+    cm_df.to_csv(cm_csv, index=True)
+
+    # --- decile lift table & CSV (rounded to 2 decimals)
+    lift_df = _decile_lift_table(y_true, y_score, pos_label=pos_label, n_bins=10)
+    lift_df_round = lift_df.copy()
+    num_cols = lift_df_round.select_dtypes(include=[np.number]).columns
+    lift_df_round[num_cols] = lift_df_round[num_cols].round(2)
+    lift_csv = os.path.join(outdir, "lift_table_deciles.csv")
+    lift_df_round.to_csv(lift_csv, index=False)
+
+    # --- plots
+    # ROC
+    if len(fpr):
+        plt.figure()
+        plt.plot(fpr, tpr, label=f"ROC (AUC={auc:.3f})")
+        plt.plot([0, 1], [0, 1], linestyle="--")
+        plt.xlabel("False Positive Rate"); plt.ylabel("True Positive Rate"); plt.title(f"{title_prefix}: ROC Curve")
+        plt.legend(loc="lower right")
+    roc_png = os.path.join(outdir, "roc_curve.png")
+    _savefig(roc_png)
+
+    # PR
+    pr, rc, _ = precision_recall_curve(y_true, y_score)
+    plt.figure()
+    plt.plot(rc, pr, label=f"PR (AP={pr_auc:.3f})")
+    plt.xlabel("Recall"); plt.ylabel("Precision"); plt.title(f"{title_prefix}: Precision–Recall Curve")
+    plt.legend(loc="lower left")
+    pr_png = os.path.join(outdir, "pr_curve.png")
+    _savefig(pr_png)
+
+    # Calibration (Reliability) curve
+    prob_true, prob_pred = calibration_curve(y_true, y_score, n_bins=10, strategy="uniform")
+    plt.figure()
+    plt.plot(prob_pred, prob_true, marker="o", label="Reliability")
+    plt.plot([0, 1], [0, 1], linestyle="--", label="Perfect")
+    plt.xlabel("Predicted probability"); plt.ylabel("Observed frequency"); plt.title(f"{title_prefix}: Calibration")
+    plt.legend(loc="upper left")
+    calib_png = os.path.join(outdir, "calibration_curve.png")
+    _savefig(calib_png)
+
+    # Lift / Gain chart (use unrounded frame for smooth curve)
+    plt.figure()
+    plt.plot(lift_df["cum_population"], lift_df["cum_gain"], marker="o", label="Cumulative Gain")
+    plt.plot([0, 1], [0, 1], linestyle="--", label="Baseline")
+    plt.xlabel("Cumulative Population"); plt.ylabel("Cumulative Gain"); plt.title(f"{title_prefix}: Cumulative Gain")
+    plt.legend(loc="lower right")
+    lift_png = os.path.join(outdir, "lift_gain_curve.png")
+    _savefig(lift_png)
+
+    # Confusion heatmap
+    plt.figure()
+    plt.imshow(cm, interpolation="nearest")
+    plt.title(f"{title_prefix}: Confusion Matrix")
+    plt.colorbar()
+    tick_marks = np.arange(2)
+    plt.xticks(tick_marks, ["Pred 0", "Pred 1"])
+    plt.yticks(tick_marks, ["Actual 0", "Actual 1"])
+    for i in range(2):
+        for j in range(2):
+            plt.text(j, i, cm[i, j], ha="center", va="center")
+    plt.ylabel("Actual"); plt.xlabel("Predicted")
+    cm_png = os.path.join(outdir, "confusion_matrix.png")
+    _savefig(cm_png)
+
+    # --- KS curve (cumulative event vs non-event by population)
+    ks_df  = _ks_curve_frame(y_true, y_score, pos_label=pos_label)
+    ks_csv = os.path.join(outdir, "ks_curve.csv")
+    ks_png = os.path.join(outdir, "ks_curve.png")
+
+    if not ks_df.empty:
+        # locate max KS point
+        ks_idx       = int(ks_df["ks_gap"].values.argmax())
+        ks_x         = float(ks_df["population"].iloc[ks_idx])
+        ks_y_event   = float(ks_df["cum_event"].iloc[ks_idx])
+        ks_y_nonevent= float(ks_df["cum_non_event"].iloc[ks_idx])
+        ks_val_annot = abs(ks_y_event - ks_y_nonevent)
+
+        # plot with explicit figure/axes only once
+        fig, ax = plt.subplots(figsize=(6, 4))
+        ax.plot(ks_df["population"], ks_df["cum_event"],     label="Cumulative Event")
+        ax.plot(ks_df["population"], ks_df["cum_non_event"], label="Cumulative Non-Event")
+
+        # vertical line & markers at max KS
+        ax.axvline(ks_x, linestyle="--", alpha=0.7)
+        ax.scatter([ks_x], [ks_y_event],    s=25)
+        ax.scatter([ks_x], [ks_y_nonevent], s=25)
+
+        # readable annotation (two lines, boxed)
+        ax.annotate(
+            f"KS = {ks_val_annot:.1%}\nPop = {ks_x:.1%}",
+            xy=(ks_x, (ks_y_event + ks_y_nonevent) / 2.0),
+            xytext=(ks_x + 0.05, min(0.9, (ks_y_event + ks_y_nonevent) / 2.0 + 0.1)),
+            arrowprops=dict(arrowstyle="->", color="black", lw=1),
+            ha="left", va="center", fontsize=10,
+            bbox=dict(boxstyle="round,pad=0.3", fc="white", ec="black", lw=0.8),
+        )
+
+        ax.set_xlabel("Population (fraction)")
+        ax.set_ylabel("Cumulative share")
+        ax.set_title(f"{title_prefix}: KS Curve")
+        ax.legend(loc="lower right")
+
+        ks_df.to_csv(ks_csv, index=False)
+        _savefig(ks_png)
+    else:
+        # Write header-only CSV and a placeholder figure so ks_png always exists
+        pd.DataFrame(columns=["population","cum_event","cum_non_event","ks_gap"]).to_csv(ks_csv, index=False)
+        fig, ax = plt.subplots(figsize=(6, 4))
+        ax.set_title(f"{title_prefix}: KS Curve (not available)")
+        _savefig(ks_png)
+
+    # --- build summaries: raw + rounded for display
+    summary = ClassificationSummary(
+        auc=float(auc) if auc == auc else np.nan,  # handle NaN
+        ks=float(ks) if ks == ks else np.nan,
+        accuracy=float(acc),
+        precision=float(precision),
+        recall=float(recall),
+        f1=float(f1),
+        pr_auc=float(pr_auc) if pr_auc == pr_auc else np.nan,
+        brier=float(brier),
+        gini=float(gini) if gini == gini else np.nan,
+        roc_png=roc_png,
+        pr_png=pr_png,
+        lift_png=lift_png,
+        calib_png=calib_png,
+        cm_png=cm_png,
+        ks_png=ks_png,                # <---- NEW
+        confusion_csv=cm_csv,
+        lift_csv=lift_csv,
+    )
+    summary_raw = asdict(summary)
+
+    # round only numeric metric fields to 2 decimals; keep paths as-is
+    metric_fields = {"auc", "ks", "accuracy", "precision", "recall", "f1", "pr_auc", "brier", "gini"}
+    summary_rounded = {}
+    for k, v in summary_raw.items():
+        if k in metric_fields:
+            try:
+                summary_rounded[k] = None if v is None or (isinstance(v, float) and np.isnan(v)) else round(float(v), 2)
+            except Exception:
+                summary_rounded[k] = v
+        else:
+            summary_rounded[k] = v
+
+    return {
+        "summary": summary_rounded,   # rounded for report/UI display
+        "summary_raw": summary_raw,   # full precision preserved for rules/debug
+        "tables": {
+            "confusion_csv": cm_csv,
+            "lift_csv": lift_csv,
+            "ks_csv": ks_csv,       
+        },
+        "plots": {
+            "roc": roc_png,
+            "pr": pr_png,
+            "lift": lift_png,
+            "calibration": calib_png,
+            "confusion": cm_png,
+            "ks": ks_png,          
+        },
+        # return rounded rows so the DOCX table looks clean
+        "deciles": lift_df_round.to_dict(orient="records"),
+    }

tanml/checks/raw_data.py

@@ -1,39 +1,75 @@
-# tanml/checks/raw_data.py  ← make sure this is the ONLY copy on disk
-from .base import BaseCheck
+from __future__ import annotations
+
+from typing import Any, Dict
 import pandas as pd
 
+from .base import BaseCheck
+from tanml.utils.data_loader import load_dataframe
+
+
 class RawDataCheck(BaseCheck):
-    def __init__(self,
-                 model, X_train, X_test, y_train, y_test,
-                 rule_config, cleaned_data,
-                 raw_data=None):
-        # bring in rule_config & cleaned_data
-        super().__init__(model, X_train, X_test, y_train, y_test,
-                         rule_config, cleaned_data)
+    """
+    Raw data sanity metrics. Accepts:
+      - DataFrame via `raw_data`, or
+      - a file path via `raw_data`, or
+      - YAML: paths.raw_data
+    """
+
+    def __init__(
+        self,
+        model,
+        X_train,
+        X_test,
+        y_train,
+        y_test,
+        rule_config: Dict[str, Any],
+        cleaned_data: pd.DataFrame,
+        raw_data: Any = None,
+    ):
+        super().__init__(model, X_train, X_test, y_train, y_test, rule_config, cleaned_data)
 
         if not hasattr(self, "config") or self.config is None:
             self.config = {}
 
         if raw_data is not None:
-            if isinstance(raw_data, (str, bytes)):
-                raw_data = pd.read_csv(raw_data)
-            if not isinstance(raw_data, pd.DataFrame):
-                raise ValueError("raw_data must be a pandas DataFrame or CSV path")
-            self.config["raw_data"] = raw_data
-
-    def run(self):
-        results = {}
+            if isinstance(raw_data, pd.DataFrame):
+                self.config["raw_data"] = raw_data
+            elif isinstance(raw_data, (str, bytes)):
+                self.config["raw_data"] = load_dataframe(raw_data)
+            else:
+                raise ValueError("raw_data must be a pandas DataFrame or a file path")
+
+    def run(self) -> Dict[str, Any]:
+        results: Dict[str, Any] = {}
         try:
-            df = self.config.get("raw_data")         
+            df = self.config.get("raw_data")
+
+            # If df is a path-like string, load it now
+            if isinstance(df, (str, bytes)):
+                df = load_dataframe(df)
+                self.config["raw_data"] = df
+
+            # If still missing, try YAML fallbacks
             if not isinstance(df, pd.DataFrame):
-                raise ValueError("raw_data not found or not a DataFrame")
+                paths = self.config.get("paths") if isinstance(self.config.get("paths"), dict) else None
+                raw_path = None
+                if paths:
+                    raw_path = paths.get("raw_data") or paths.get("raw")
+                raw_path = raw_path or self.config.get("raw_data_path") or self.config.get("raw_path")
 
-            results["total_rows"]   = int(df.shape[0])
+                if raw_path:
+                    df = load_dataframe(raw_path)
+                    self.config["raw_data"] = df
+                else:
+                    raise ValueError("raw_data not found: provide DataFrame or set paths.raw_data in YAML")
+
+            # --- metrics ---
+            results["total_rows"] = int(df.shape[0])
             results["total_columns"] = int(df.shape[1])
 
             miss = df.isnull().mean().round(4)
-            results["avg_missing"]            = float(miss.mean())
-            results["columns_with_missing"]   = miss[miss > 0].to_dict()
+            results["avg_missing"] = float(miss.mean())
+            results["columns_with_missing"] = miss[miss > 0].to_dict()
 
             results["duplicate_rows"] = int(df.duplicated().sum())
 
@@ -44,4 +80,3 @@ class RawDataCheck(BaseCheck):
             results["error"] = str(e)
 
         return {"RawDataCheck": results}
-

tanml/checks/regression_metrics.py

@@ -0,0 +1,167 @@
+# tanml/checks/regression_metrics.py
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Tuple
+import numpy as np
+
+try:
+    # Prefer sklearn implementations when available
+    from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error
+    _HAS_SKLEARN = True
+except Exception:
+    _HAS_SKLEARN = False
+
+from .base import BaseCheck
+
+
+class RegressionMetricsCheck(BaseCheck):
+    """
+    Computes TanML's frozen regression metrics:
+      - RMSE
+      - MAE
+      - Median Absolute Error
+      - R²
+      - Adjusted R²
+      - MAPE (or SMAPE fallback when zeros/near-zeros exist in y_true)
+
+    Pure compute: no file I/O, no plotting. Returns a dict.
+    """
+
+    def __init__(
+        self,
+        *,
+        y_true: np.ndarray,
+        y_pred: np.ndarray,
+        n_features: Optional[int] = None,
+        mape_eps: float = 1e-8,
+        config: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """
+        Parameters
+        ----------
+        y_true : array-like of shape (n_samples,)
+        y_pred : array-like of shape (n_samples,)
+        n_features : int, optional
+            Number of model features (for Adjusted R²). If None, Adjusted R² may be None.
+        mape_eps : float
+            Small constant to guard divisions in MAPE/SMAPE.
+        config : dict, optional
+            Reserved for future options.
+        """
+        self.y_true = np.asarray(y_true).reshape(-1)
+        self.y_pred = np.asarray(y_pred).reshape(-1)
+        self.n_features = int(n_features) if n_features is not None else None
+        self.mape_eps = float(mape_eps)
+        self.config = config or {}
+
+        self._notes: List[str] = []
+
+    # ---------------------------
+    # Public API
+    # ---------------------------
+    def run(self) -> Dict[str, Any]:
+        self._validate_inputs()
+
+        rmse = self._rmse(self.y_true, self.y_pred)
+        mae = self._mae(self.y_true, self.y_pred)
+        median_ae = self._median_ae(self.y_true, self.y_pred)
+
+        r2, r2_adj = self._r2_and_adjusted(self.y_true, self.y_pred, self.n_features)
+
+        mape_val, smape_val, used = self._mape_or_smape(self.y_true, self.y_pred, self.mape_eps)
+
+        return {
+            "rmse": rmse,
+            "mae": mae,
+            "median_ae": median_ae,
+            "r2": r2,
+            "r2_adjusted": r2_adj,
+            "mape_or_smape": mape_val if used == "MAPE" else smape_val,
+            "mape_used": (used == "MAPE"),
+            "notes": self._notes,  # human-readable notes (e.g., SMAPE fallback, zero variance)
+        }
+
+    # ---------------------------
+    # Internals
+    # ---------------------------
+    def _validate_inputs(self) -> None:
+        if self.y_true.shape != self.y_pred.shape:
+            raise ValueError(f"Shapes differ: y_true{self.y_true.shape} vs y_pred{self.y_pred.shape}")
+        if self.y_true.ndim != 1 or self.y_pred.ndim != 1:
+            raise ValueError("y_true and y_pred must be 1-D arrays.")
+        if self.y_true.size < 2:
+            self._notes.append("Too few samples (<2) — some metrics may be undefined.")
+
+    @staticmethod
+    def _rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float:
+        if _HAS_SKLEARN:
+            return float(np.sqrt(mean_squared_error(y_true, y_pred)))
+        return float(np.sqrt(np.mean((y_true - y_pred) ** 2)))
+
+    @staticmethod
+    def _mae(y_true: np.ndarray, y_pred: np.ndarray) -> float:
+        if _HAS_SKLEARN:
+            return float(mean_absolute_error(y_true, y_pred))
+        return float(np.mean(np.abs(y_true - y_pred)))
+
+    @staticmethod
+    def _median_ae(y_true: np.ndarray, y_pred: np.ndarray) -> float:
+        return float(np.median(np.abs(y_true - y_pred)))
+
+    def _r2_and_adjusted(
+        self,
+        y_true: np.ndarray,
+        y_pred: np.ndarray,
+        n_features: Optional[int],
+    ) -> Tuple[Optional[float], Optional[float]]:
+        n = int(y_true.size)
+        # If variance is zero, R² is undefined
+        if float(np.var(y_true)) == 0.0:
+            self._notes.append("R² undefined: target has zero variance.")
+            return None, None
+
+        if _HAS_SKLEARN:
+            try:
+                r2_val = float(r2_score(y_true, y_pred))
+            except Exception:
+                r2_val = None
+                self._notes.append("R² could not be computed via sklearn.r2_score.")
+        else:
+            ss_res = float(np.sum((y_true - y_pred) ** 2))
+            ss_tot = float(np.sum((y_true - np.mean(y_true)) ** 2))
+            r2_val = float(1.0 - ss_res / ss_tot) if ss_tot > 0 else None
+
+        if r2_val is None or n_features is None:
+            return r2_val, None
+
+        p = int(n_features)
+        if n <= p + 1:
+            self._notes.append("Adjusted R² unavailable: insufficient degrees of freedom (n <= p + 1).")
+            return r2_val, None
+
+        try:
+            r2_adj = float(1.0 - (1.0 - r2_val) * (n - 1) / (n - p - 1))
+        except Exception:
+            r2_adj = None
+            self._notes.append("Adjusted R² computation failed due to numeric issues.")
+        return r2_val, r2_adj
+
+    def _mape_or_smape(
+        self, y_true: np.ndarray, y_pred: np.ndarray, eps: float
+    ) -> Tuple[Optional[float], Optional[float], str]:
+        """Return (MAPE, SMAPE, used_flag) and record notes for fallbacks."""
+        has_near_zero = np.any(np.abs(y_true) <= eps)
+        smape_val = self._smape(y_true, y_pred, eps)
+
+        if has_near_zero:
+            self._notes.append("MAPE skipped due to zeros/near-zeros in target; SMAPE reported instead.")
+            return None, smape_val, "SMAPE"
+
+        mape_val = float(np.mean(np.abs((y_true - y_pred) / (y_true + eps))) * 100.0)
+        return mape_val, smape_val, "MAPE"
+
+    @staticmethod
+    def _smape(y_true: np.ndarray, y_pred: np.ndarray, eps: float) -> float:
+        num = 2.0 * np.abs(y_pred - y_true)
+        den = np.abs(y_true) + np.abs(y_pred) + eps
+        return float(np.mean(num / den) * 100.0)