SuperModelingFactory 0.3.5__tar.gz → 0.3.7__tar.gz

{supermodelingfactory-0.3.5 → supermodelingfactory-0.3.7}/Modeling_Tool/Explainability/Model_Explainer.py

@@ -22,6 +22,7 @@ from __future__ import annotations
 import ast
 import importlib.metadata
 import sys
+import warnings
 
 import numpy as np
 import pandas as pd
@@ -848,6 +849,125 @@ class ModelExplainer:
     # ------------------------------------------------------------------ #
     # LIME
     # ------------------------------------------------------------------ #
+    @staticmethod
+    def _lime_missing_mask(frame: pd.DataFrame) -> pd.Series:
+        numeric = frame.select_dtypes(include=[np.number])
+        if numeric.empty:
+            return pd.Series(False, index=frame.index)
+        return numeric.isna().any(axis=1)
+
+    @classmethod
+    def _warn_lime_missing(
+        cls,
+        frame: pd.DataFrame,
+        *,
+        role: str,
+        missing_strategy: str,
+        n_before: int,
+        n_after: int,
+        fill_values: dict[str, float] | None = None,
+    ) -> None:
+        numeric = frame.select_dtypes(include=[np.number])
+        if numeric.empty:
+            return
+        null_cols = [col for col in numeric.columns if numeric[col].isna().any()]
+        if not null_cols:
+            return
+        details = []
+        for col in null_cols:
+            count = int(numeric[col].isna().sum())
+            pct = 100.0 * count / len(frame) if len(frame) else 0.0
+            detail = f"{col}: {count}/{len(frame)} ({pct:.1f}%)"
+            if fill_values and col in fill_values:
+                detail += f", median_fill={fill_values[col]:.6g}"
+            details.append(detail)
+        msg = (
+            f"LIME missing_strategy={missing_strategy!r} on {role}: "
+            f"null columns={null_cols}; " + "; ".join(details)
+        )
+        if missing_strategy == "drop":
+            msg += f"; rows {n_before} -> {n_after}"
+        warnings.warn(msg, UserWarning, stacklevel=3)
+
+    @classmethod
+    def _prepare_lime_data(
+        cls,
+        train: pd.DataFrame,
+        rows: pd.DataFrame | None = None,
+        *,
+        train_role: str = "X_train",
+        rows_role: str = "x_row",
+        missing_strategy: str = "median",
+    ) -> tuple[pd.DataFrame, pd.DataFrame | None]:
+        if missing_strategy not in {"median", "drop"}:
+            raise ValueError("missing_strategy must be 'median' or 'drop'")
+
+        train_clean = train.copy()
+        rows_clean = rows.copy() if rows is not None else None
+        numeric_cols = list(train_clean.select_dtypes(include=[np.number]).columns)
+
+        if missing_strategy == "median":
+            fill_values: dict[str, float] = {}
+            for col in numeric_cols:
+                if train_clean[col].isna().any() or (
+                    rows_clean is not None and rows_clean[col].isna().any()
+                ):
+                    median_val = float(train_clean[col].median())
+                    fill_values[col] = median_val
+                    train_clean[col] = train_clean[col].fillna(median_val)
+                    if rows_clean is not None:
+                        rows_clean[col] = rows_clean[col].fillna(median_val)
+            cls._warn_lime_missing(
+                train,
+                role=train_role,
+                missing_strategy=missing_strategy,
+                n_before=len(train),
+                n_after=len(train_clean),
+                fill_values=fill_values or None,
+            )
+            if rows_clean is not None:
+                cls._warn_lime_missing(
+                    rows,
+                    role=rows_role,
+                    missing_strategy=missing_strategy,
+                    n_before=len(rows),
+                    n_after=len(rows_clean),
+                    fill_values=fill_values or None,
+                )
+            return train_clean, rows_clean
+
+        n_train_before = len(train_clean)
+        train_clean = train_clean.loc[~cls._lime_missing_mask(train_clean)].copy()
+        cls._warn_lime_missing(
+            train,
+            role=train_role,
+            missing_strategy=missing_strategy,
+            n_before=n_train_before,
+            n_after=len(train_clean),
+        )
+        if train_clean.empty:
+            raise ValueError(
+                f"LIME missing_strategy='drop' removed all rows from {train_role} "
+                f"({n_train_before} -> 0). Provide complete training data or use missing_strategy='median'."
+            )
+
+        if rows_clean is not None:
+            n_rows_before = len(rows_clean)
+            rows_clean = rows_clean.loc[~cls._lime_missing_mask(rows_clean)].copy()
+            cls._warn_lime_missing(
+                rows,
+                role=rows_role,
+                missing_strategy=missing_strategy,
+                n_before=n_rows_before,
+                n_after=len(rows_clean),
+            )
+            if rows_clean.empty:
+                raise ValueError(
+                    f"LIME missing_strategy='drop' removed all rows from {rows_role} "
+                    f"({n_rows_before} -> 0). Impute missing values or use missing_strategy='median'."
+                )
+        return train_clean, rows_clean
+
     def _build_lime_explainer(self, X_train, num_features=None, random_state=None, **lime_kwargs):
         LimeTabularExplainer = _lazy_lime()
         train = self._as_frame(X_train if X_train is not None else self.background_data)
@@ -864,7 +984,16 @@ class ModelExplainer:
             **lime_kwargs,
         )
 
-    def lime_explain_instance(self, x_row, X_train=None, num_features=10, num_samples=5000, random_state=None, **lime_kwargs):
+    def lime_explain_instance(
+        self,
+        x_row,
+        X_train=None,
+        num_features=10,
+        num_samples=5000,
+        random_state=None,
+        missing_strategy="median",
+        **lime_kwargs,
+    ):
         """Explain one sample with LIME."""
         if isinstance(x_row, pd.Series):
             x_row = x_row.to_frame().T
@@ -874,7 +1003,14 @@ class ModelExplainer:
         if frame.shape[0] != 1:
             frame = frame.iloc[[0]]
 
-        train = X_train if X_train is not None else self.background_data
+        train_raw = self._as_frame(X_train if X_train is not None else self.background_data)
+        train, frame = self._prepare_lime_data(
+            train_raw,
+            frame,
+            train_role="X_train",
+            rows_role="x_row",
+            missing_strategy=missing_strategy,
+        )
         explainer = self._build_lime_explainer(train, num_features, random_state, **lime_kwargs)
         explanation = explainer.explain_instance(
             data_row=np.asarray(frame.iloc[0]),
@@ -896,17 +1032,36 @@ class ModelExplainer:
         out.attrs["score"] = explanation.score
         return out
 
-    def lime_global_importance(self, X, X_train=None, num_features=10, num_samples=2000, sample_size=100, random_state=None, **lime_kwargs):
+    def lime_global_importance(
+        self,
+        X,
+        X_train=None,
+        num_features=10,
+        num_samples=2000,
+        sample_size=100,
+        random_state=None,
+        missing_strategy="median",
+        **lime_kwargs,
+    ):
         """Aggregate LIME local weights across a sample as global importance."""
+        train_raw = self._as_frame(X_train if X_train is not None else self.background_data)
         frame = self._sample_frame(X, sample_size=sample_size, random_state=random_state)
+        train, frame = self._prepare_lime_data(
+            train_raw,
+            frame,
+            train_role="X_train",
+            rows_role="X",
+            missing_strategy=missing_strategy,
+        )
         rows = []
         for _, row in frame.iterrows():
             local = self.lime_explain_instance(
                 row,
-                X_train=X_train,
+                X_train=train,
                 num_features=num_features,
                 num_samples=num_samples,
                 random_state=random_state,
+                missing_strategy="median",
                 **lime_kwargs,
             )
             rows.append(local)

supermodelingfactory-0.3.7/Modeling_Tool/Pipeline/_common.py

@@ -0,0 +1,366 @@
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import fields, is_dataclass
+from pathlib import Path
+from typing import Any, Iterable, Mapping
+
+import numpy as np
+import pandas as pd
+
+_QUERY_COLUMN_RE = re.compile(r"(?<!['\"])\b([A-Za-z_][\w]*)\b(?![\'\"])")
+_QUERY_RESERVED = frozenset(
+    {
+        "and",
+        "or",
+        "not",
+        "True",
+        "False",
+        "None",
+        "in",
+        "is",
+        "where",
+        "if",
+        "else",
+        "lambda",
+        "abs",
+        "all",
+        "any",
+        "max",
+        "min",
+        "sum",
+        "len",
+        "round",
+        "float",
+        "int",
+        "str",
+        "bool",
+    }
+)
+
+
+def make_dirs(*paths: str | os.PathLike | None) -> None:
+    for path in paths:
+        if path:
+            os.makedirs(str(path), exist_ok=True)
+
+
+def as_list(value: Any | Iterable[Any] | None, default: list[Any] | None = None) -> list[Any]:
+    if value is None:
+        return list(default or [])
+    if isinstance(value, str):
+        return [value]
+    return list(value)
+
+
+def merge_dict(base: Mapping[str, Any] | None, override: Mapping[str, Any] | None) -> dict[str, Any]:
+    merged = dict(base or {})
+    merged.update(dict(override or {}))
+    return merged
+
+
+def query_referenced_columns(expr: str) -> set[str]:
+    """Return identifier-like column names referenced in a pandas query expression."""
+    if not expr or not str(expr).strip():
+        return set()
+    return {
+        match.group(1)
+        for match in _QUERY_COLUMN_RE.finditer(str(expr))
+        if match.group(1) not in _QUERY_RESERVED
+    }
+
+
+def validate_woe_fit_query_syntax(data: pd.DataFrame, query: str) -> None:
+    """Raise ValueError when a woe_fit_query expression is syntactically invalid."""
+    if not query:
+        return
+    sample = data.head(min(100, len(data)))
+    if sample.empty:
+        raise ValueError("woe_fit_query cannot be validated on an empty dataset.")
+    try:
+        sample.query(query, engine="python")
+    except SyntaxError as exc:
+        raise ValueError(f"Invalid woe_fit_query syntax: {query!r}") from exc
+    except Exception as exc:
+        raise ValueError(f"woe_fit_query failed on sample data: {query!r} ({exc})") from exc
+
+
+def validate_woe_fit_query_columns(
+    query: str,
+    available_cols: Iterable[str],
+    *,
+    context: str = "dataset",
+) -> None:
+    """Raise KeyError when query references columns missing from available_cols."""
+    if not query:
+        return
+    available = set(available_cols)
+    missing = sorted(query_referenced_columns(query) - available)
+    if missing:
+        raise KeyError(
+            f"woe_fit_query references missing columns in {context}: {missing}. "
+            "Add them to the input data, batch_base_cols, or population_dims."
+        )
+
+
+def apply_woe_fit_query(
+    train: pd.DataFrame,
+    query: str | None,
+    *,
+    target: str | None = None,
+) -> tuple[pd.DataFrame, dict[str, Any] | None]:
+    """Filter WOE fit rows via pandas query and return an audit row for refine_summary."""
+    if not query:
+        return train, None
+    n_before = len(train)
+    try:
+        filtered = train.query(query, engine="python").copy()
+    except Exception as exc:
+        audit = {
+            "target": target,
+            "step": "fit_filter",
+            "status": "error",
+            "query": query,
+            "n_before": n_before,
+            "error": repr(exc),
+        }
+        return train, audit
+    audit = {
+        "target": target,
+        "step": "fit_filter",
+        "status": "ok",
+        "query": query,
+        "n_before": n_before,
+        "n_after": len(filtered),
+    }
+    return filtered, audit
+
+
+def dataclass_to_dict(obj: Any) -> dict[str, Any]:
+    if not is_dataclass(obj):
+        return dict(obj)
+    return {field.name: getattr(obj, field.name) for field in fields(obj)}
+
+
+def safe_to_csv(df: pd.DataFrame | None, path: str | os.PathLike, index: bool = False) -> None:
+    if df is None:
+        return
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    df.to_csv(path, index=index)
+
+
+def persist_explain_outputs(
+    outputs: Mapping[str, Any],
+    explain_dir: str | os.PathLike,
+) -> dict[str, dict[str, str]]:
+    """Write explain_outputs tables to ``explain_dir`` and return path index."""
+    explain_root = Path(explain_dir)
+    explain_root.mkdir(parents=True, exist_ok=True)
+    manifest_rows: list[dict[str, str]] = []
+    explain_paths: dict[str, dict[str, str]] = {}
+
+    if "import_error" in outputs:
+        err_path = explain_root / "import_error.txt"
+        err_path.write_text(str(outputs["import_error"]), encoding="utf-8")
+        manifest_rows.append(
+            {"model": "_global", "artifact": "import_error", "path": str(err_path), "status": "error"}
+        )
+
+    for model_name, payload in outputs.items():
+        if model_name == "import_error":
+            continue
+        if not isinstance(payload, dict):
+            continue
+        model_dir = explain_root / str(model_name)
+        paths: dict[str, str] = {}
+
+        if payload.get("error") and "feature_importance" not in payload:
+            err_path = model_dir / "error.txt"
+            model_dir.mkdir(parents=True, exist_ok=True)
+            err_path.write_text(str(payload["error"]), encoding="utf-8")
+            paths["error"] = str(err_path)
+            manifest_rows.append(
+                {
+                    "model": str(model_name),
+                    "artifact": "error",
+                    "path": str(err_path),
+                    "status": "error",
+                }
+            )
+            explain_paths[str(model_name)] = paths
+            continue
+
+        fi = payload.get("feature_importance")
+        if isinstance(fi, pd.DataFrame) and not fi.empty:
+            fi_path = model_dir / "feature_importance.csv"
+            safe_to_csv(fi, fi_path)
+            paths["feature_importance"] = str(fi_path)
+            manifest_rows.append(
+                {
+                    "model": str(model_name),
+                    "artifact": "feature_importance",
+                    "path": str(fi_path),
+                    "status": "ok",
+                }
+            )
+
+        owen = payload.get("owen")
+        if isinstance(owen, dict):
+            if owen.get("error"):
+                err_path = model_dir / "owen_error.txt"
+                model_dir.mkdir(parents=True, exist_ok=True)
+                err_path.write_text(str(owen["error"]), encoding="utf-8")
+                paths["owen_error"] = str(err_path)
+                manifest_rows.append(
+                    {
+                        "model": str(model_name),
+                        "artifact": "owen_error",
+                        "path": str(err_path),
+                        "status": "error",
+                    }
+                )
+            for artifact, filename in (
+                ("feature_importance", "owen_feature_importance.csv"),
+                ("group_importance", "owen_group_importance.csv"),
+            ):
+                table = owen.get(artifact)
+                if isinstance(table, pd.DataFrame) and not table.empty:
+                    table_path = model_dir / filename
+                    safe_to_csv(table, table_path)
+                    key = f"owen_{artifact}"
+                    paths[key] = str(table_path)
+                    manifest_rows.append(
+                        {
+                            "model": str(model_name),
+                            "artifact": key,
+                            "path": str(table_path),
+                            "status": "ok",
+                        }
+                    )
+
+        for artifact_key in ("shap_summary", "plot_error"):
+            plot_path = payload.get(artifact_key)
+            if plot_path and artifact_key == "shap_summary":
+                paths["shap_summary"] = str(plot_path)
+                manifest_rows.append(
+                    {
+                        "model": str(model_name),
+                        "artifact": "shap_summary",
+                        "path": str(plot_path),
+                        "status": "ok",
+                    }
+                )
+            elif plot_path and artifact_key == "plot_error":
+                err_path = model_dir / "plot_error.txt"
+                model_dir.mkdir(parents=True, exist_ok=True)
+                err_path.write_text(str(plot_path), encoding="utf-8")
+                paths["plot_error"] = str(err_path)
+                manifest_rows.append(
+                    {
+                        "model": str(model_name),
+                        "artifact": "plot_error",
+                        "path": str(err_path),
+                        "status": "plot_error",
+                    }
+                )
+
+        if paths:
+            explain_paths[str(model_name)] = paths
+
+    if manifest_rows:
+        manifest_df = pd.DataFrame(manifest_rows)
+        safe_to_csv(manifest_df, explain_root / "explain_manifest.csv", index=False)
+        explain_paths.setdefault("_manifest", {})["explain_manifest"] = str(explain_root / "explain_manifest.csv")
+
+    return explain_paths
+
+
+def get_raw_model(model: Any) -> Any:
+    if hasattr(model, "_model") and hasattr(model._model, "model"):
+        return model._model.model
+    if hasattr(model, "model"):
+        return model.model
+    return model
+
+
+def predict_positive(model: Any, data: pd.DataFrame, feature_cols: list[str]) -> np.ndarray:
+    raw_model = get_raw_model(model)
+    x = data[feature_cols]
+
+    if hasattr(model, "_model") and hasattr(raw_model, "predict_proba"):
+        pred = raw_model.predict_proba(x)
+    elif model.__class__.__name__ == "LRMaster" and hasattr(model, "predict_proba"):
+        pred = model.predict_proba(data, varlist=feature_cols)
+    elif hasattr(model, "predict_proba"):
+        try:
+            pred = model.predict_proba(x)
+        except TypeError:
+            pred = model.predict_proba(data, varlist=feature_cols)
+    elif hasattr(raw_model, "predict_proba"):
+        pred = raw_model.predict_proba(x)
+    elif hasattr(model, "predict"):
+        pred = model.predict(x)
+    elif hasattr(raw_model, "predict"):
+        pred = raw_model.predict(x)
+    else:
+        raise TypeError(f"Model {type(model)!r} does not expose predict/predict_proba")
+
+    pred_arr = np.asarray(pred)
+    if pred_arr.ndim == 2 and pred_arr.shape[1] > 1:
+        return pred_arr[:, 1]
+    return pred_arr.reshape(-1)
+
+
+def add_dataset_with_optional_weight(
+    evaluator: Any,
+    name: str,
+    data: pd.DataFrame,
+    weight_col: str | None = None,
+) -> None:
+    if weight_col:
+        try:
+            evaluator.add_dataset(name, data, weight_col=weight_col)
+            return
+        except TypeError:
+            pass
+    evaluator.add_dataset(name, data)
+
+
+def write_basic_excel(
+    report_path: str,
+    sheets: Mapping[str, pd.DataFrame | None],
+    title: str | None = None,
+) -> str:
+    from ExcelMaster.ExcelMaster import ExcelMaster
+
+    Path(report_path).parent.mkdir(parents=True, exist_ok=True)
+    em = ExcelMaster(report_path, verbose=False)
+    used: set[str] = set()
+
+    def sheet_name(name: str) -> str:
+        clean = str(name).replace("/", "_").replace("\\", "_").replace(":", "-")
+        clean = clean.replace("[", "(").replace("]", ")").replace("*", "_").replace("?", "_")
+        clean = clean.strip("' ") or "Sheet"
+        base = clean[:31]
+        candidate = base
+        i = 2
+        while candidate.lower() in used:
+            suffix = f"_{i}"
+            candidate = f"{base[:31 - len(suffix)]}{suffix}"
+            i += 1
+        used.add(candidate.lower())
+        return candidate
+
+    if title:
+        ws = em.add_worksheet(sheet_name("Overview"))
+        em.write_text_content(ws, input_text=title)
+
+    for name, df in sheets.items():
+        if df is None:
+            continue
+        ws = em.add_worksheet(sheet_name(name))
+        em.write_dataframe(ws, df=df, title=str(name), index=False)
+
+    em.close_workbook()
+    return report_path