trusttab 0.1.0__py3-none-any.whl

trusttab/__init__.py

@@ -0,0 +1,5 @@
+"""TrustTab: Reliability-aware stress testing for tabular machine learning."""
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__"]

trusttab/api.py

@@ -0,0 +1,82 @@
+"""Supported convenience API for common TrustTab workflows.
+
+Specialized modules remain importable for advanced use. Names listed here are
+the stable, curated entry points intended for applications and examples.
+"""
+
+from trusttab.data import (
+    build_preprocessor,
+    load_adult,
+    load_german_credit,
+    load_heart_disease,
+)
+from trusttab.evaluation import (
+    adaptive_expected_calibration_error,
+    calibrate_classifier,
+    calibrate_fitted_classifier,
+    evaluate_classifier,
+    expected_calibration_error,
+    log_loss_score,
+    maximum_calibration_error,
+)
+from trusttab.experiments import (
+    load_experiment_config,
+    run_robustness_benchmark,
+    run_stress_test,
+)
+from trusttab.experiments.config import expand_experiment_conditions
+from trusttab.models import SUPPORTED_MODELS, build_model
+from trusttab.noise import (
+    apply_feature_shifts,
+    inject_symmetric_label_noise,
+    validate_feature_shifts,
+)
+from trusttab.reporting import generate_robustness_report
+from trusttab.statistics import (
+    analyze_friedman_nemenyi,
+    calibration_wilcoxon_results,
+    compare_degradation_slopes,
+    descriptive_summary,
+    run_research_analysis,
+)
+from trusttab.visualization import (
+    plot_metric_comparison,
+    plot_probability_distribution,
+    plot_reliability_diagram,
+    plot_roc_curve,
+    save_figure,
+)
+
+__all__ = [
+    "SUPPORTED_MODELS",
+    "adaptive_expected_calibration_error",
+    "analyze_friedman_nemenyi",
+    "apply_feature_shifts",
+    "build_model",
+    "build_preprocessor",
+    "calibrate_classifier",
+    "calibrate_fitted_classifier",
+    "calibration_wilcoxon_results",
+    "compare_degradation_slopes",
+    "descriptive_summary",
+    "evaluate_classifier",
+    "expand_experiment_conditions",
+    "expected_calibration_error",
+    "generate_robustness_report",
+    "inject_symmetric_label_noise",
+    "load_adult",
+    "load_experiment_config",
+    "load_german_credit",
+    "load_heart_disease",
+    "log_loss_score",
+    "maximum_calibration_error",
+    "plot_metric_comparison",
+    "plot_probability_distribution",
+    "plot_reliability_diagram",
+    "plot_roc_curve",
+    "run_research_analysis",
+    "run_robustness_benchmark",
+    "run_stress_test",
+    "save_figure",
+    "validate_feature_shifts",
+]

trusttab/data/__init__.py

@@ -0,0 +1,28 @@
+"""Dataset loading utilities for TrustTab."""
+
+from trusttab.data.preprocessing import build_preprocessor
+
+from trusttab.data.german_credit import (
+    load_german_credit,
+    validate_german_credit_data,
+)
+
+from trusttab.data.adult import (
+    load_adult,
+    validate_adult_data,
+)
+
+from trusttab.data.heart_disease import (
+    load_heart_disease,
+    validate_heart_disease_data,
+)
+
+__all__ = [
+    "load_german_credit",
+    "validate_german_credit_data",
+    "load_adult",
+    "validate_adult_data",
+    "load_heart_disease",
+    "validate_heart_disease_data",
+    "build_preprocessor",
+]

trusttab/data/adult.py

@@ -0,0 +1,117 @@
+
+"""Adult Census Income dataset loading and validation."""
+
+from pathlib import Path
+
+import pandas as pd
+from sklearn.datasets import fetch_openml
+
+
+OPENML_DATA_ID = 1590
+EXPECTED_ROWS = 48842
+EXPECTED_FEATURES = 14
+
+TARGET_MAPPING = {
+    "<=50k": 0,
+    ">50k": 1,
+}
+
+
+def validate_adult_data(
+    features: pd.DataFrame,
+    target: pd.Series,
+) -> None:
+    """Validate the structure and contents of the Adult dataset."""
+
+    if not isinstance(features, pd.DataFrame):
+        raise TypeError("Features must be a pandas DataFrame.")
+
+    if not isinstance(target, pd.Series):
+        raise TypeError("Target must be a pandas Series.")
+
+    if features.empty:
+        raise ValueError("The feature dataset is empty.")
+
+    if len(features) != len(target):
+        raise ValueError("Features and target have different row counts.")
+
+    if features.shape[0] != EXPECTED_ROWS:
+        raise ValueError(
+            f"Expected {EXPECTED_ROWS} rows, got {features.shape[0]}."
+        )
+
+    if features.shape[1] != EXPECTED_FEATURES:
+        raise ValueError(
+            f"Expected {EXPECTED_FEATURES} features, "
+            f"got {features.shape[1]}."
+        )
+
+    if features.columns.has_duplicates:
+        raise ValueError("Feature names must be unique.")
+
+    if target.isnull().any():
+        raise ValueError("Target contains missing values.")
+
+    observed_labels = (
+        set(target.astype(str).str.strip().str.lower().unique())
+    )
+
+    if observed_labels != set(TARGET_MAPPING):
+        raise ValueError(
+            "Unexpected target labels. "
+            f"Expected {set(TARGET_MAPPING)}, got {observed_labels}."
+        )
+
+
+def load_adult(
+    data_home: str | Path | None = None,
+) -> tuple[pd.DataFrame, pd.Series]:
+    """
+    Download and load the Adult Census Income dataset from OpenML.
+
+    Returns
+    -------
+    X:
+        DataFrame containing 14 input features. Missing feature
+        values are preserved for downstream preprocessing.
+    y:
+        Integer target Series: <=50K=0, >50K=1.
+    """
+
+    dataset = fetch_openml(
+        data_id=OPENML_DATA_ID,
+        as_frame=True,
+        data_home=data_home,
+    )
+
+    features = dataset.data.copy()
+    raw_target = dataset.target.copy()
+
+    # Convert question-mark missing-value markers to actual NaN values.
+    features = features.replace(
+        r"^\s*\?\s*$",
+        float("nan"),
+        regex=True,
+    )
+
+    # Remove surrounding whitespace from categorical values.
+    for column in features.select_dtypes(
+        include=["object", "category"]
+    ).columns:
+        features[column] = features[column].map(
+            lambda value: value.strip() if isinstance(value, str) else value
+        )
+
+    validate_adult_data(features, raw_target)
+
+    target = (
+        raw_target.astype(str)
+        .str.strip()
+        .str.lower()
+        .map(TARGET_MAPPING)
+        .astype("int64")
+    )
+
+    target.name = "target"
+
+    return features, target

trusttab/data/german_credit.py

@@ -0,0 +1,102 @@
+
+"""German Credit dataset loading and validation."""
+
+from pathlib import Path
+
+import pandas as pd
+from sklearn.datasets import fetch_openml
+
+
+OPENML_DATA_ID = 31
+EXPECTED_ROWS = 1000
+EXPECTED_FEATURES = 20
+
+TARGET_MAPPING = {
+    "good": 0,
+    "bad": 1,
+}
+
+
+def validate_german_credit_data(
+    features: pd.DataFrame,
+    target: pd.Series,
+) -> None:
+    """Validate the structure and contents of the German Credit dataset."""
+
+    if not isinstance(features, pd.DataFrame):
+        raise TypeError("Features must be a pandas DataFrame.")
+
+    if not isinstance(target, pd.Series):
+        raise TypeError("Target must be a pandas Series.")
+
+    if features.empty:
+        raise ValueError("The feature dataset is empty.")
+
+    if len(features) != len(target):
+        raise ValueError("Features and target have different row counts.")
+
+    if features.shape[0] != EXPECTED_ROWS:
+        raise ValueError(
+            f"Expected {EXPECTED_ROWS} rows, got {features.shape[0]}."
+        )
+
+    if features.shape[1] != EXPECTED_FEATURES:
+        raise ValueError(
+            f"Expected {EXPECTED_FEATURES} features, "
+            f"got {features.shape[1]}."
+        )
+
+    if features.columns.has_duplicates:
+        raise ValueError("Feature names must be unique.")
+
+    if features.isnull().any().any():
+        raise ValueError("Features contain missing values.")
+
+    if target.isnull().any():
+        raise ValueError("Target contains missing values.")
+
+    observed_labels = set(target.astype(str).str.strip().str.lower().unique())
+
+    if observed_labels != set(TARGET_MAPPING):
+        raise ValueError(
+            "Unexpected target labels. "
+            f"Expected {set(TARGET_MAPPING)}, got {observed_labels}."
+        )
+
+
+def load_german_credit(
+    data_home: str | Path | None = None,
+) -> tuple[pd.DataFrame, pd.Series]:
+    """
+    Download and load the German Credit dataset from OpenML.
+
+    Returns
+    -------
+    X:
+        DataFrame containing 20 input features.
+    y:
+        Integer target Series: good=0, bad=1.
+    """
+
+    dataset = fetch_openml(
+        data_id=OPENML_DATA_ID,
+        as_frame=True,
+        data_home=data_home,
+    )
+
+    features = dataset.data.copy()
+    raw_target = dataset.target.copy()
+
+    validate_german_credit_data(features, raw_target)
+
+    target = (
+        raw_target.astype(str)
+        .str.strip()
+        .str.lower()
+        .map(TARGET_MAPPING)
+        .astype("int64")
+    )
+
+    target.name = "target"
+
+    return features, target

trusttab/data/heart_disease.py

@@ -0,0 +1,229 @@
+"""UCI Heart Disease processed Cleveland dataset loader.
+
+Source: UCI Machine Learning Repository, dataset 45, processed Cleveland
+subset only. Citation: Janosi, A., Steinbrunn, W., Pfisterer, M., & Detrano,
+R. (1989). *Heart Disease*. https://doi.org/10.24432/C52P4X. The dataset is
+licensed CC BY 4.0; see https://creativecommons.org/licenses/by/4.0/.
+
+The source ``num`` target is retained as ``y.attrs['original_num']`` before
+binary mapping (0 -> 0, 1-4 -> 1). Feature missing values are preserved.
+"""
+
+from io import BytesIO
+from pathlib import Path
+from urllib.request import urlopen
+from zipfile import ZipFile
+
+import numpy as np
+import pandas as pd
+
+UCI_DATA_ID = 45
+DATASET_URL = "https://archive.ics.uci.edu/static/public/45/heart+disease.zip"
+EXPECTED_ROWS = 303
+EXPECTED_FEATURES = 13
+CACHE_FILENAME = "heart_disease_uci45_cleveland.csv"
+
+FEATURE_COLUMNS = (
+    "age",
+    "sex",
+    "cp",
+    "trestbps",
+    "chol",
+    "fbs",
+    "restecg",
+    "thalach",
+    "exang",
+    "oldpeak",
+    "slope",
+    "ca",
+    "thal",
+)
+TARGET_COLUMN = "num"
+CATEGORICAL_FEATURES = (
+    "sex",
+    "cp",
+    "fbs",
+    "restecg",
+    "exang",
+    "slope",
+    "ca",
+    "thal",
+)
+MISSING_VALUE_FEATURES = ("ca", "thal")
+TARGET_MAPPING = {0: 0, 1: 1, 2: 1, 3: 1, 4: 1}
+EXPECTED_NUM_COUNTS = {0: 164, 1: 55, 2: 36, 3: 35, 4: 13}
+
+
+def _read_processed_cleveland_archive(archive: bytes) -> pd.DataFrame:
+    """Read only UCI's processed Cleveland member from its dataset archive."""
+
+    with ZipFile(BytesIO(archive)) as zipped:
+        matches = [
+            name
+            for name in zipped.namelist()
+            if Path(name).name.lower() == "processed.cleveland.data"
+        ]
+        if len(matches) != 1:
+            raise ValueError(
+                "UCI dataset archive must contain exactly one "
+                "processed.cleveland.data file."
+            )
+        with zipped.open(matches[0]) as source_file:
+            frame = pd.read_csv(
+                source_file,
+                header=None,
+                names=[*FEATURE_COLUMNS, TARGET_COLUMN],
+                na_values=["?"],
+                skipinitialspace=True,
+            )
+    return frame
+
+
+def _download_processed_cleveland() -> pd.DataFrame:
+    """Download UCI dataset 45 and parse only the processed Cleveland file."""
+
+    with urlopen(DATASET_URL, timeout=30) as response:
+        return _read_processed_cleveland_archive(response.read())
+
+
+def validate_heart_disease_data(
+    features: pd.DataFrame,
+    original_num: pd.Series,
+) -> None:
+    """Validate processed Cleveland rows, schema, missingness, and ``num``.
+
+    The UCI source has missing values in ``ca`` and ``thal``. They are
+    intentionally preserved for downstream train-only imputation; rows are
+    never dropped and no imputation occurs in this loader.
+    """
+
+    if not isinstance(features, pd.DataFrame):
+        raise TypeError("Features must be a pandas DataFrame.")
+    if not isinstance(original_num, pd.Series):
+        raise TypeError("Original num target must be a pandas Series.")
+    if features.empty:
+        raise ValueError("The feature dataset is empty.")
+    if len(features) != len(original_num):
+        raise ValueError("Features and target have different row counts.")
+    if features.shape[0] != EXPECTED_ROWS:
+        raise ValueError(f"Expected {EXPECTED_ROWS} rows, got {features.shape[0]}.")
+    if features.shape[1] != EXPECTED_FEATURES:
+        raise ValueError(
+            f"Expected {EXPECTED_FEATURES} features, got {features.shape[1]}."
+        )
+    if tuple(features.columns) != FEATURE_COLUMNS:
+        raise ValueError("Feature columns do not match the processed Cleveland schema.")
+    if features.columns.has_duplicates:
+        raise ValueError("Feature names must be unique.")
+    if original_num.isna().any():
+        raise ValueError("Original num target contains missing values.")
+
+    raw_values = pd.to_numeric(original_num, errors="coerce").to_numpy(dtype=float)
+    if not np.isfinite(raw_values).all():
+        raise ValueError("Original num target must contain finite numeric values.")
+    if not np.equal(raw_values, np.floor(raw_values)).all():
+        raise ValueError("Original num target values must be integers 0 through 4.")
+    observed_targets = set(raw_values.astype(int))
+    if not observed_targets.issubset(TARGET_MAPPING):
+        raise ValueError(
+            "Unexpected original num target values. Expected a subset of "
+            f"{set(TARGET_MAPPING)}, got {observed_targets}."
+        )
+    observed_counts = pd.Series(raw_values.astype(int)).value_counts().sort_index()
+    if observed_counts.to_dict() != EXPECTED_NUM_COUNTS:
+        raise ValueError(
+            "Unexpected processed Cleveland num class distribution. Expected "
+            f"{EXPECTED_NUM_COUNTS}, got {observed_counts.to_dict()}."
+        )
+
+    allowed_missing = set(MISSING_VALUE_FEATURES)
+    missing_columns = set(features.columns[features.isna().any()])
+    if missing_columns.difference(allowed_missing):
+        raise ValueError(
+            "Unexpected missing feature values in columns: "
+            f"{sorted(missing_columns.difference(allowed_missing))}."
+        )
+
+    for column in features.columns:
+        converted = pd.to_numeric(features[column], errors="coerce")
+        invalid_values = features[column].notna() & converted.isna()
+        if invalid_values.any():
+            raise ValueError(f"Feature '{column}' contains non-numeric values.")
+        if not np.isfinite(converted.dropna().to_numpy(dtype=float)).all():
+            raise ValueError(f"Feature '{column}' contains non-finite values.")
+
+
+def _normalize_features(features: pd.DataFrame) -> pd.DataFrame:
+    """Set stable numeric and categorical dtypes without filling missing data."""
+
+    normalized = features.loc[:, list(FEATURE_COLUMNS)].copy()
+    for column in FEATURE_COLUMNS:
+        normalized[column] = pd.to_numeric(normalized[column], errors="raise")
+    for column in CATEGORICAL_FEATURES:
+        codes = normalized[column].astype("Int64").astype(object)
+        normalized[column] = codes.where(codes.notna(), np.nan)
+    return normalized
+
+
+def load_heart_disease(
+    data_home: str | Path | None = None,
+) -> tuple[pd.DataFrame, pd.Series]:
+    """Load UCI dataset 45's processed Cleveland subset.
+
+    Only ``processed.cleveland.data`` is read from the UCI archive; the
+    Cleveland, Hungarian, Switzerland, and VA datasets are never combined.
+    With ``data_home``, the original 13 feature columns and multiclass ``num``
+    are cached as CSV and reused on later calls. ``?`` source markers become
+    ``NaN``. The 303 rows are retained and feature missingness is left for the
+    experiment pipeline to impute after splitting.
+
+    The returned target is binary: original ``num`` 0 maps to 0 and values
+    1, 2, 3, or 4 map to 1. The unmodified numeric target is also preserved in
+    ``y.attrs['original_num']``. UCI citation and attribution: Janosi et al.
+    (1989), *Heart Disease*, DOI: https://doi.org/10.24432/C52P4X, CC BY 4.0.
+
+    Parameters
+    ----------
+    data_home:
+        Optional cache directory. If omitted, the UCI archive is fetched for
+        each call and no local dataset file is written.
+
+    Returns
+    -------
+    X:
+        DataFrame with the 13 processed Cleveland features in documented
+        order. Categorical codes use object dtype with integer values; numeric
+        measurements remain numeric. Missing values are preserved as NaN.
+    y:
+        Integer binary target Series named ``target`` with original ``num``
+        available in its ``attrs`` metadata.
+    """
+
+    cache_path = Path(data_home) / CACHE_FILENAME if data_home is not None else None
+    if cache_path is not None and cache_path.exists():
+        raw_data = pd.read_csv(cache_path, na_values=["?"])
+        missing_columns = set([*FEATURE_COLUMNS, TARGET_COLUMN]).difference(
+            raw_data.columns
+        )
+        if missing_columns:
+            raise ValueError(
+                f"Cached Cleveland data is missing columns: {sorted(missing_columns)}."
+            )
+        features = raw_data.loc[:, list(FEATURE_COLUMNS)].copy()
+        original_num = raw_data[TARGET_COLUMN].copy()
+    else:
+        raw_data = _download_processed_cleveland()
+        if cache_path is not None:
+            cache_path.parent.mkdir(parents=True, exist_ok=True)
+            raw_data.to_csv(cache_path, index=False)
+        features = raw_data.loc[:, list(FEATURE_COLUMNS)].copy()
+        original_num = raw_data[TARGET_COLUMN].copy()
+
+    validate_heart_disease_data(features, original_num)
+    normalized_features = _normalize_features(features)
+    numeric_num = pd.to_numeric(original_num, errors="raise").astype("int64")
+    target = numeric_num.map(TARGET_MAPPING).astype("int64")
+    target.name = "target"
+    target.attrs["original_num"] = numeric_num.copy()
+
+    return normalized_features, target

trusttab/data/preprocessing.py

@@ -0,0 +1,69 @@
+"""Leakage-safe preprocessing utilities for tabular datasets."""
+
+import pandas as pd
+from sklearn.compose import ColumnTransformer
+from sklearn.impute import SimpleImputer
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import OneHotEncoder, StandardScaler
+
+
+def build_preprocessor(
+    features: pd.DataFrame, *, scale_numeric: bool = True
+) -> ColumnTransformer:
+    """
+    Build an unfitted preprocessing transformer for tabular features.
+
+    Numeric columns:
+        Median imputation, optionally followed by standard scaling.
+
+    Categorical columns:
+        Most-frequent imputation followed by one-hot encoding.
+
+    The returned transformer must be fitted only on training data.
+    """
+
+    if not isinstance(features, pd.DataFrame):
+        raise TypeError("Features must be a pandas DataFrame.")
+
+    if features.empty:
+        raise ValueError("Features must not be empty.")
+
+    if features.columns.has_duplicates:
+        raise ValueError("Feature names must be unique.")
+
+    numeric_columns = features.select_dtypes(include=["number"]).columns.tolist()
+
+    categorical_columns = [
+        column for column in features.columns if column not in numeric_columns
+    ]
+
+    if not numeric_columns and not categorical_columns:
+        raise ValueError("No usable feature columns found.")
+
+    transformers = []
+
+    if numeric_columns:
+        numeric_steps = [("imputer", SimpleImputer(strategy="median"))]
+        if scale_numeric:
+            numeric_steps.append(("scaler", StandardScaler()))
+        numeric_pipeline = Pipeline(steps=numeric_steps)
+
+        transformers.append(("numeric", numeric_pipeline, numeric_columns))
+
+    if categorical_columns:
+        categorical_pipeline = Pipeline(
+            steps=[
+                ("imputer", SimpleImputer(strategy="most_frequent")),
+                (
+                    "onehot",
+                    OneHotEncoder(handle_unknown="ignore"),
+                ),
+            ]
+        )
+
+        transformers.append(("categorical", categorical_pipeline, categorical_columns))
+
+    return ColumnTransformer(
+        transformers=transformers,
+        remainder="drop",
+    )

trusttab/evaluation/__init__.py

@@ -0,0 +1,31 @@
+"""Evaluation utilities for TrustTab."""
+
+from trusttab.evaluation.calibration import (
+    calibrate_classifier,
+    calibrate_fitted_classifier,
+    get_calibration_curve,
+)
+from trusttab.evaluation.metrics import (
+    adaptive_expected_calibration_error,
+    evaluate_classifier,
+    expected_calibration_error,
+    log_loss_score,
+    maximum_calibration_error,
+)
+from trusttab.evaluation.operating_region import (
+    OperatingPoint,
+    analyze_operating_region,
+)
+
+__all__ = [
+    "OperatingPoint",
+    "adaptive_expected_calibration_error",
+    "analyze_operating_region",
+    "calibrate_classifier",
+    "calibrate_fitted_classifier",
+    "evaluate_classifier",
+    "expected_calibration_error",
+    "get_calibration_curve",
+    "log_loss_score",
+    "maximum_calibration_error",
+]