tanml 0.1.6__py3-none-any.whl

tanml/checks/input_cluster.py

@@ -0,0 +1,109 @@
+# -----------------------------------------------------------------------------
+# File: tanml/checks/input_cluster.py
+# -----------------------------------------------------------------------------
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import pandas as pd
+from sklearn.cluster import KMeans
+from sklearn.preprocessing import StandardScaler
+
+
+class InputClusterCoverageCheck:
+    """Cluster the model inputs and report their distribution.
+
+    On run() this class will:
+    • auto‑select k (or respect the YAML‑supplied one)
+    • save a cluster_distribution.csv  and a input_cluster_plot.png
+    • return a result‑dict consumed by the report builder.
+    """
+
+    def __init__(
+        self,
+        cleaned_df: pd.DataFrame,
+        feature_names: list[str],
+        rule_config: dict | None = None,
+    ) -> None:
+        self.cleaned_df = cleaned_df
+        self.feature_names = feature_names
+
+        cfg = (rule_config or {}).get("InputClusterCoverageCheck", {})
+        self.n_clusters: int | None = cfg.get("n_clusters")  # None → auto
+        self.max_k: int = cfg.get("max_k", 10)  # elbow upper bound
+
+ 
+    def _auto_select_k(self, X_scaled: pd.DataFrame) -> int:
+        """Simple elbow‑method heuristic."""
+        inertias: list[float] = []
+        for k in range(1, self.max_k + 1):
+            km = KMeans(n_clusters=k, n_init=10, random_state=42)
+            km.fit(X_scaled)
+            inertias.append(km.inertia_)
+
+        deltas = pd.Series(inertias).diff().iloc[1:]
+        second_delta = deltas.diff().iloc[1:]
+        if second_delta.empty:
+            return 2  # fall‑back if data are too small
+        return second_delta.idxmax() + 1  # +1 because diff() shifts index
+
+ 
+    def run(self) -> dict:
+        missing = set(self.feature_names) - set(self.cleaned_df.columns)
+        if missing:
+            raise ValueError(f"Features not in cleaned_df: {missing}")
+
+        X = self.cleaned_df[self.feature_names]
+        X_scaled = StandardScaler().fit_transform(X)
+
+        if self.n_clusters is None:
+            self.n_clusters = self._auto_select_k(X_scaled)
+
+        km = KMeans(n_clusters=self.n_clusters, n_init=10, random_state=42)
+        labels = km.fit_predict(X_scaled)
+
+        dfc = X.copy()
+        dfc["cluster"] = labels
+
+        summary = (
+            dfc["cluster"].value_counts().sort_index().reset_index(name="Count")
+        )
+        summary.columns = ["Cluster", "Count"]
+        total = len(dfc)
+        summary["Percent"] = (summary["Count"] / total * 100).round(2)
+
+        Path("reports/clusters").mkdir(parents=True, exist_ok=True)
+        Path("reports/images").mkdir(parents=True, exist_ok=True)
+
+        csv_path = "reports/clusters/cluster_distribution.csv"
+        plot_path = "reports/images/input_cluster_plot.png"
+
+        summary.to_csv(csv_path, index=False)
+        self._save_bar_chart(summary, plot_path)
+
+        return {
+            "cluster_table": summary.to_dict(orient="records"),
+            "cluster_csv": csv_path,
+            "cluster_plot_img": plot_path,
+            "n_clusters": self.n_clusters,
+        }
+
+    @staticmethod
+    def _save_bar_chart(summary: pd.DataFrame, plot_path: str) -> None:
+        """Helper used internally and by tests."""
+        fig, ax = plt.subplots(figsize=(6, 4))
+        bars = ax.barh(y=summary["Cluster"].astype(str), width=summary["Count"])
+        ax.set_xlabel("Count")
+        ax.set_ylabel("Cluster")
+        ax.set_title("Input Cluster Distribution")
+        max_count = summary["Count"].max() or 0
+        for bar in bars:
+            w = bar.get_width()
+            ax.text(
+                w + max_count * 0.01,
+                bar.get_y() + bar.get_height() / 2,
+                f"{int(w)}",
+                va="center",
+            )
+        fig.tight_layout()
+        fig.savefig(plot_path)
+        plt.close(fig)

tanml/checks/logit_stats.py

@@ -0,0 +1,59 @@
+# tanml/checks/logit_stats.py
+
+import statsmodels.api as sm
+
+class LogisticStatsCheck:
+    def __init__(self, model, X, y, config):
+        self.model = model
+        self.X = X
+        self.y = y
+        self.config = config or {}
+
+    def run(self):
+        try:
+            # 1) Add constant and fit the statsmodels Logit
+            Xc = sm.add_constant(self.X, has_constant='add')
+            res = sm.Logit(self.y, Xc).fit(disp=False)
+
+            # 2) Extract coefficient table
+            coef    = res.params
+            stderr  = res.bse
+            zscore  = coef / stderr
+            pvals   = res.pvalues
+
+            table = []
+            for feat in coef.index:
+                label = "Intercept" if feat.lower() == "const" else feat
+                table.append({
+                    "feature":     label,
+                    "coefficient": float(coef[feat]),
+                    "std_error":   float(stderr[feat]),
+                    "z_score":     float(zscore[feat]),
+                    "p_value":     float(pvals[feat]),
+                })
+
+            # 3) Fit statistics
+            fit = {
+                "log_lik":   float(res.llf),
+                "aic":       float(res.aic),
+                "bic":       float(res.bic),
+                "pseudo_r2": float(res.prsquared),
+            }
+
+            # 4) Full summary text
+            summary = res.summary().as_text()
+
+            return {
+                "table":   table,
+                "fit":     fit,
+                "summary": summary,
+                "object":  res
+            }
+
+        except Exception as e:
+            return {
+                "table":   [],
+                "fit":     {},
+                "summary": f"LogisticStatsCheck failed: {e}",
+                "object":  None
+            }

tanml/checks/model_contents.py

@@ -0,0 +1,40 @@
+# tanml/checks/model_contents.py
+
+import inspect
+
+class ModelContentsCheck:
+    def __init__(self, model):
+        self.model = model
+
+    def run(self):
+        summary = {}
+
+        # 1. Model type and module
+        summary["model_class"] = type(self.model).__name__
+        summary["module"] = type(self.model).__module__
+
+        # 2. Feature names
+        if hasattr(self.model, "feature_names_in_"):
+            summary["feature_names_in"] = list(self.model.feature_names_in_)
+
+        # 3. Hyperparameters
+        if hasattr(self.model, "get_params"):
+            try:
+                summary["hyperparameters"] = self.model.get_params()
+            except Exception:
+                summary["hyperparameters"] = "Could not extract"
+
+        # 4. Coefficients
+        if hasattr(self.model, "coef_"):
+            try:
+                summary["coefficients"] = self.model.coef_.tolist()
+            except Exception:
+                summary["coefficients"] = "Could not extract"
+
+        # 5. Public attributes
+        summary["attributes"] = [
+            name for name in dir(self.model)
+            if not name.startswith("_") and not inspect.ismethod(getattr(self.model, name))
+        ]
+
+        return summary

tanml/checks/model_meta.py

@@ -0,0 +1,50 @@
+from .base import BaseCheck
+import pandas as pd
+
+class ModelMetaCheck(BaseCheck):
+    def __init__(self, model, X_train, X_test, y_train, y_test, rule_config, cleaned_data):
+        super().__init__(model, X_train, X_test, y_train, y_test, rule_config, cleaned_data)
+
+    def run(self):
+        result = {}
+        try:
+            # Basic metadata
+            result["model_type"] = type(self.model).__name__
+            result["model_class"] = type(self.model).__name__
+            result["module"] = getattr(self.model, "__module__", "Unknown")
+
+            # Features
+            result["n_features"] = self.X_train.shape[1]
+            result["feature_names"] = list(getattr(self.X_train, "columns", []))
+
+            # Training stats
+            result["n_train_rows"] = self.X_train.shape[0]
+            y_series = pd.Series(self.y_train)
+            result["target_balance"] = y_series.value_counts().to_dict()
+
+            # Hyperparameters
+            try:
+                params = self.model.get_params()
+                result["hyperparam_table"] = [
+                    {"param": k, "value": str(v)} for k, v in params.items()
+                ]
+            except Exception as e:
+                result["hyperparam_table"] = [{"param": "error", "value": str(e)}]
+
+            # Public attributes
+            try:
+                result["attributes"] = {
+                    k: str(v)
+                    for k, v in self.model.__dict__.items()
+                    if not k.startswith("_")
+                }
+            except Exception as e:
+                result["attributes"] = {"error": str(e)}
+
+            result["status"] = "Model metadata extracted successfully"
+
+        except Exception as e:
+            result["error"] = str(e)
+            result["status"] = "ModelMetaCheck failed"
+
+        return result

tanml/checks/performance.py

@@ -0,0 +1,90 @@
+from sklearn.metrics import accuracy_score, roc_auc_score, f1_score, confusion_matrix
+from scipy.stats import ks_2samp
+from .base import BaseCheck
+import numpy as np
+
+class PerformanceCheck(BaseCheck):
+    def __init__(self, model, X_train, X_test, y_train, y_test, rule_config, cleaned_data):
+        super().__init__(model, X_train, X_test, y_train, y_test, rule_config, cleaned_data)
+        self.model = model
+        self.X_train = X_train
+        self.X_test = X_test
+        self.y_train = y_train
+        self.y_test = y_test
+        self.rule_config = rule_config or {}
+
+    def run(self):
+        """Compute metrics from model predictions on provided test set."""
+        result = {
+            "accuracy":          self.compute_accuracy(),
+            "auc_roc":           self.compute_auc(),
+            "f1":                self.compute_f1(),
+            "ks":                self.compute_ks(),
+            "confusion_matrix":  self.compute_confusion(),
+        }
+        result["auc"] = result["auc_roc"]  # alias for backward compatibility
+        return result
+
+    def compute_accuracy(self):
+        y_pred = self.model.predict(self.X_test)
+        return round(accuracy_score(self.y_test, y_pred), 4)
+
+    def compute_auc(self):
+        y_prob = self.model.predict_proba(self.X_test)[:, 1]
+        return round(roc_auc_score(self.y_test, y_prob), 4)
+
+    def compute_f1(self):
+        y_pred = self.model.predict(self.X_test)
+        return round(f1_score(self.y_test, y_pred), 4)
+
+    def compute_ks(self):
+        y_prob = self.model.predict_proba(self.X_test)[:, 1]
+        y_true = self.y_test
+
+        # Split by class
+        prob_0 = y_prob[y_true == 0]
+        prob_1 = y_prob[y_true == 1]
+
+        if len(prob_0) < 2 or len(prob_1) < 2:
+            return "Insufficient data for KS test"
+
+        return round(ks_2samp(prob_0, prob_1).statistic, 4)
+
+    def compute_confusion(self):
+        y_pred = self.model.predict(self.X_test)
+        return confusion_matrix(self.y_test, y_pred).tolist()
+
+    @staticmethod
+    def from_predictions(y_true, y_proba):
+        """
+        Compute metrics directly from true labels and predicted probs.
+        Returns a dict: accuracy, auc, f1, ks, confusion_matrix.
+        """
+        y_true_arr = np.array(y_true)
+        y_proba_arr = np.array(y_proba)
+        y_pred = (y_proba_arr >= 0.5).astype(int)
+
+        # Avoid invalid metrics when only one class present
+        if len(np.unique(y_true_arr)) < 2:
+            return {
+                "accuracy": "N/A", "auc": "N/A", "f1": "N/A",
+                "ks": "N/A", "confusion_matrix": []
+            }
+
+        result = {
+            "accuracy":         round(accuracy_score(y_true_arr, y_pred), 4),
+            "auc_roc":          round(roc_auc_score(y_true_arr, y_proba_arr), 4),
+            "f1":               round(f1_score(y_true_arr, y_pred), 4),
+            "confusion_matrix": confusion_matrix(y_true_arr, y_pred).tolist(),
+        }
+
+        prob_0 = y_proba_arr[y_true_arr == 0]
+        prob_1 = y_proba_arr[y_true_arr == 1]
+
+        result["ks"] = (
+            round(ks_2samp(prob_0, prob_1).statistic, 4)
+            if len(prob_0) >= 2 and len(prob_1) >= 2 else "N/A"
+        )
+
+        result["auc"] = result["auc_roc"]  # alias for template/report
+        return result

tanml/checks/raw_data.py

@@ -0,0 +1,47 @@
+# tanml/checks/raw_data.py  ← make sure this is the ONLY copy on disk
+from .base import BaseCheck
+import pandas as pd
+
+class RawDataCheck(BaseCheck):
+    def __init__(self,
+                 model, X_train, X_test, y_train, y_test,
+                 rule_config, cleaned_data,
+                 raw_data=None):
+        # bring in rule_config & cleaned_data
+        super().__init__(model, X_train, X_test, y_train, y_test,
+                         rule_config, cleaned_data)
+
+        if not hasattr(self, "config") or self.config is None:
+            self.config = {}
+
+        if raw_data is not None:
+            if isinstance(raw_data, (str, bytes)):
+                raw_data = pd.read_csv(raw_data)
+            if not isinstance(raw_data, pd.DataFrame):
+                raise ValueError("raw_data must be a pandas DataFrame or CSV path")
+            self.config["raw_data"] = raw_data
+
+    def run(self):
+        results = {}
+        try:
+            df = self.config.get("raw_data")         
+            if not isinstance(df, pd.DataFrame):
+                raise ValueError("raw_data not found or not a DataFrame")
+
+            results["total_rows"]   = int(df.shape[0])
+            results["total_columns"] = int(df.shape[1])
+
+            miss = df.isnull().mean().round(4)
+            results["avg_missing"]            = float(miss.mean())
+            results["columns_with_missing"]   = miss[miss > 0].to_dict()
+
+            results["duplicate_rows"] = int(df.duplicated().sum())
+
+            const_cols = [c for c in df.columns if df[c].nunique(dropna=False) <= 1]
+            results["constant_columns"] = const_cols
+
+        except Exception as e:
+            results["error"] = str(e)
+
+        return {"RawDataCheck": results}
+

tanml/checks/rule_engine.py

@@ -0,0 +1,45 @@
+from .base import BaseCheck
+
+class RuleEngineCheck(BaseCheck):
+    def run(self):
+        result = {}
+        try:
+            perf = self.rule_config.get("check_results", {}).get("PerformanceCheck", {})
+            rules = self.rule_config.get("rules", {})
+
+            applied_rules = {}
+
+            for metric, conditions in rules.items():
+                actual_value = perf.get(metric)
+
+                if actual_value is None:
+                    applied_rules[metric] = "❌ Metric not found"
+                    continue
+
+                # Attempt to cast to float
+                try:
+                    actual_value = float(actual_value)
+                except (ValueError, TypeError):
+                    applied_rules[metric] = f"❌ Invalid value: {actual_value}"
+                    continue
+
+                # Evaluate min/max
+                rule_messages = []
+                passed = True
+
+                if "min" in conditions and actual_value < conditions["min"]:
+                    passed = False
+                    rule_messages.append(f"{actual_value:.4f} < min {conditions['min']}")
+                if "max" in conditions and actual_value > conditions["max"]:
+                    passed = False
+                    rule_messages.append(f"{actual_value:.4f} > max {conditions['max']}")
+
+                applied_rules[metric] = "✅ Passed" if passed else "❌ " + " | ".join(rule_messages)
+
+            result["rules"] = applied_rules
+            result["overall_pass"] = all(v.startswith("✅") for v in applied_rules.values())
+
+        except Exception as e:
+            result["error"] = str(e)
+
+        return result

tanml/checks/stress_test.py

@@ -0,0 +1,64 @@
+from sklearn.metrics import roc_auc_score, accuracy_score
+import numpy as np
+import pandas as pd
+
+class StressTestCheck:
+    def __init__(self, model, X, y, epsilon=0.01, perturb_fraction=0.2):
+        self.model = model
+        self.X = X.copy()
+        self.y = y
+        self.epsilon = epsilon
+        self.perturb_fraction = perturb_fraction
+
+    def run(self):
+        np.random.seed(42)
+        results = []
+
+        # Compute baseline metrics
+        try:
+            base_proba = self.model.predict_proba(self.X)[:, 1]
+            base_pred = (base_proba >= 0.5).astype(int)
+            base_auc = roc_auc_score(self.y, base_proba)
+            base_acc = accuracy_score(self.y, base_pred)
+        except Exception as e:
+            print(f"⚠️ Error computing baseline metrics: {e}")
+            return []
+
+        # Perturb each numeric feature
+        for col in self.X.columns:
+            if not pd.api.types.is_numeric_dtype(self.X[col]):
+                continue  # skip non-numeric features
+
+            try:
+                n_perturb = int(self.perturb_fraction * len(self.X))
+                idx = np.random.choice(self.X.index, size=n_perturb, replace=False)
+
+                X_perturbed = self.X.copy()
+                X_perturbed.loc[idx, col] += self.epsilon
+
+                perturbed_proba = self.model.predict_proba(X_perturbed)[:, 1]
+                perturbed_pred = (perturbed_proba >= 0.5).astype(int)
+
+                pert_auc = roc_auc_score(self.y, perturbed_proba)
+                pert_acc = accuracy_score(self.y, perturbed_pred)
+
+                results.append({
+                    "feature": col,
+                    "perturbation": f"±{round(self.epsilon * 100, 2)}%",
+                    "accuracy": round(pert_acc, 4),
+                    "auc": round(pert_auc, 4),
+                    "delta_accuracy": round(pert_acc - base_acc, 4),
+                    "delta_auc": round(pert_auc - base_auc, 4),
+                })
+
+            except Exception as e:
+                results.append({
+                    "feature": col,
+                    "perturbation": f"±{round(self.epsilon * 100, 2)}%",
+                    "accuracy": "error",
+                    "auc": "error",
+                    "delta_accuracy": f"Error: {e}",
+                    "delta_auc": f"Error: {e}",
+                })
+
+        return pd.DataFrame(results)

tanml/checks/vif.py

@@ -0,0 +1,51 @@
+import pandas as pd
+import os
+from statsmodels.stats.outliers_influence import variance_inflation_factor
+
+class VIFCheck:
+    def __init__(self, model, X_train, X_test, y_train, y_test, rule_config, cleaned_df, output_dir="reports/vif"):
+        self.cleaned_df = cleaned_df.select_dtypes(include=['float64', 'int64']).dropna()
+        self.output_dir = output_dir
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def run(self):
+        result = {}
+        try:
+            if self.cleaned_df.shape[1] < 2:
+                return {
+                    "vif_table": [],
+                    "high_vif_features": [],
+                    "status": "Not enough numeric features"
+                }
+
+            X = self.cleaned_df.copy()
+            X.insert(0, "Intercept", 1)  # Add constant term for VIF
+
+            vif_data = []
+            for i in range(X.shape[1]):
+                try:
+                    vif = variance_inflation_factor(X.values, i)
+                except Exception:
+                    vif = float("inf")
+                vif_data.append({
+                    "feature": X.columns[i],
+                    "VIF": round(vif, 2)
+                })
+
+            high_vif = [row["feature"] for row in vif_data if row["feature"] != "Intercept" and row["VIF"] > 5]
+
+            # Save to CSV
+            output_path = os.path.join(self.output_dir, "vif_table.csv")
+            pd.DataFrame(vif_data).to_csv(output_path, index=False)
+
+            result["vif_table"] = vif_data
+            result["high_vif_features"] = high_vif
+            result["csv_path"] = output_path
+            result["status"] = "VIF computed successfully"
+
+        except Exception as e:
+            result["vif_table"] = []
+            result["high_vif_features"] = []
+            result["status"] = f"VIFCheck failed: {str(e)}"
+
+        return result

tanml/cli/arg_parser.py

@@ -0,0 +1,31 @@
+import argparse
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Run TanML model validation toolkit")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+
+    validate_parser = subparsers.add_parser("validate", help="Run validation checks and generate report")
+    validate_parser.add_argument("--model", required=False,
+                        help="Model path: .pkl for sklearn/xgb, .csv for SAS or R logistic")
+    validate_parser.add_argument("--raw", required=False, help="Path to raw input data file")
+    validate_parser.add_argument("--cleaned", required=False, help="Path to cleaned input data file")
+    validate_parser.add_argument("--rules", required=True, help="Path to rules.yaml config file")
+    validate_parser.add_argument("--target", required=False, help="Target column name (optional)")
+    validate_parser.add_argument("--features", required=False, help="Comma-separated list of features")
+    validate_parser.add_argument(
+        "--report_path",
+        type=str,
+        default="reports/final_report.docx",
+        help="Path to output DOCX report. Example: --report_path my_reports/run1.docx"
+    )
+
+
+    init_parser = subparsers.add_parser("init", help="Generate starter rules.yaml file")
+    init_parser.add_argument("--scenario", required=True, choices=["A", "B", "C"],
+                             help="Choose validation scenario: A (single model), B (multiple segments), C (single dataset + segment column)")
+    init_parser.add_argument(
+        "--output", type=str, default="rules.yaml",
+        help="Destination path for rules YAML file (default: rules.yaml)"
+    )
+    return parser.parse_args()

tanml/cli/init_cmd.py

@@ -0,0 +1,8 @@
+from tanml.utils.yaml_generator import generate_rules_yaml
+
+def run_init(scenario, dest_path="rules.yaml", overwrite=False):
+    try:
+        generate_rules_yaml(scenario=scenario, dest_path=dest_path, overwrite=overwrite)
+
+    except Exception as e:
+        print(f"❌ Failed to create YAML: {e}")

tanml/cli/main.py

@@ -0,0 +1,27 @@
+import argparse
+from tanml.cli.validate_cmd import run_validate
+from tanml.cli.init_cmd import run_init
+
+def main():
+    parser = argparse.ArgumentParser(prog="tanml")
+    subparsers = parser.add_subparsers(dest="command")
+
+    # tanml validate --rules path.yaml
+    validate_parser = subparsers.add_parser("validate", help="Run model validation")
+    validate_parser.add_argument("--rules", required=True, help="Path to rules/config YAML")
+
+    # tanml init --scenario B
+    init_parser = subparsers.add_parser("init", help="Initialize rules YAML template")
+    init_parser.add_argument("--scenario", choices=["A", "B", "C"], required=True, help="Scenario type")
+    init_parser.add_argument("--overwrite", action="store_true", help="Overwrite existing rules.yaml if it exists")
+    init_parser.add_argument("--output", default="rules.yaml", help="Path where rules.yaml should be saved (default: rules.yaml)")
+
+    args = parser.parse_args()
+
+    if args.command == "validate":
+        run_validate(args.rules)
+    elif args.command == "init":
+           run_init(args.scenario, dest_path=args.output, overwrite=args.overwrite)
+
+    else:
+        parser.print_help()

tanml/cli/validate_cmd.py

@@ -0,0 +1,7 @@
+# tanml/cli/validate_cmd.py
+
+from tanml.validate import validate_from_yaml
+
+def run_validate(rules_path):
+    print(f"🧪 Starting validation using rules from: {rules_path}")
+    validate_from_yaml(rules_path)