ensurv 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ensurv/__init__.py +6 -0
- ensurv/_diagnostics.py +99 -0
- ensurv/_random.py +39 -0
- ensurv/cross_validation.py +229 -0
- ensurv/diagnostics.py +336 -0
- ensurv/evaluation.py +171 -0
- ensurv/horizon.py +104 -0
- ensurv/km.py +191 -0
- ensurv/loss.py +95 -0
- ensurv/model.py +1029 -0
- ensurv/preprocessing.py +301 -0
- ensurv/utils.py +441 -0
- ensurv/workflows.py +280 -0
- ensurv-0.1.0.dist-info/METADATA +69 -0
- ensurv-0.1.0.dist-info/RECORD +17 -0
- ensurv-0.1.0.dist-info/WHEEL +5 -0
- ensurv-0.1.0.dist-info/top_level.txt +1 -0
ensurv/__init__.py
ADDED
ensurv/_diagnostics.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Internal parameter-group and structural-weight diagnostics."""
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import torch.nn as nn
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def _get_param_groups(net):
|
|
8
|
+
r"""Collect parameter names into semantic groups for training diagnostics.
|
|
9
|
+
|
|
10
|
+
:param net: Network whose linear pathway, latent projection, hidden layers, normalization
|
|
11
|
+
layers, noise transform, and output layer are inspected.
|
|
12
|
+
:type net: _EnSurvNet
|
|
13
|
+
|
|
14
|
+
:returns: Group labels mapped to names used by ``net.named_parameters()``.
|
|
15
|
+
:rtype: dict[str, list[str]]
|
|
16
|
+
"""
|
|
17
|
+
groups = {}
|
|
18
|
+
if net.apply_linear:
|
|
19
|
+
groups['beta (linear)'] = ['beta']
|
|
20
|
+
if net.W is not None:
|
|
21
|
+
groups['W (latent proj)'] = ['W']
|
|
22
|
+
for i in range(net.num_layer):
|
|
23
|
+
keys = [f'layers.{i}.weight', f'layers.{i}.bias']
|
|
24
|
+
if net.bns is not None:
|
|
25
|
+
keys += [f'bns.{i}.weight', f'bns.{i}.bias']
|
|
26
|
+
groups[f'layer {i}'] = keys
|
|
27
|
+
if net.noise_transform is not None:
|
|
28
|
+
groups['noise transform'] = [name for name, _ in net.named_parameters() if name.startswith('noise_transform.')]
|
|
29
|
+
groups['final_layer'] = ['final_layer.weight', 'final_layer.bias']
|
|
30
|
+
return groups
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _group_grad_norms(net, param_groups):
|
|
34
|
+
r"""Compute the root mean square of available gradients in each parameter group.
|
|
35
|
+
|
|
36
|
+
:param net: Network with gradients populated by the current backward pass.
|
|
37
|
+
:type net: _EnSurvNet
|
|
38
|
+
:param param_groups: Semantic group labels and corresponding named parameters, as returned
|
|
39
|
+
by ``_get_param_groups``.
|
|
40
|
+
:type param_groups: dict[str, list[str]]
|
|
41
|
+
|
|
42
|
+
:returns: One RMS gradient magnitude per group; groups without gradients receive zero.
|
|
43
|
+
:rtype: dict[str, float]
|
|
44
|
+
|
|
45
|
+
Call after ``loss.backward()`` and before clearing gradients. For a group with
|
|
46
|
+
:math:`N_g` available gradient entries, the reported value is
|
|
47
|
+
:math:`\{\sum_{j=1}^{N_g} (\nabla_j L)^2 / \max(N_g,1)\}^{1/2}`.
|
|
48
|
+
"""
|
|
49
|
+
named = dict(net.named_parameters())
|
|
50
|
+
out = {}
|
|
51
|
+
for gname, keys in param_groups.items():
|
|
52
|
+
sq, cnt = 0.0, 0
|
|
53
|
+
for k in keys:
|
|
54
|
+
p = named.get(k, None)
|
|
55
|
+
if p is not None and p.grad is not None:
|
|
56
|
+
sq += p.grad.detach().pow(2).sum().item()
|
|
57
|
+
cnt += p.grad.numel()
|
|
58
|
+
out[gname] = (sq / max(cnt, 1)) ** 0.5
|
|
59
|
+
return out
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _extract_raw_importance(net, input_dim):
|
|
63
|
+
r"""Extract descriptive linear coefficients and nonlinear structural norms.
|
|
64
|
+
|
|
65
|
+
:param net: Network whose current parameter values are inspected without a forward pass.
|
|
66
|
+
:type net: _EnSurvNet
|
|
67
|
+
:param input_dim: Number of covariates; used to select feature columns from the first
|
|
68
|
+
hidden-layer weight matrix.
|
|
69
|
+
:type input_dim: int
|
|
70
|
+
|
|
71
|
+
:returns: Linear coefficients, or None when disabled, and one nonlinear norm per feature.
|
|
72
|
+
:rtype: tuple[numpy.ndarray or None, numpy.ndarray]
|
|
73
|
+
|
|
74
|
+
With a latent projection, use row norms of its weight matrix. Otherwise multiply compatible
|
|
75
|
+
absolute hidden-layer weight matrices and take feature-column norms.
|
|
76
|
+
"""
|
|
77
|
+
beta_weights = None
|
|
78
|
+
if net.apply_linear:
|
|
79
|
+
beta_weights = net.beta.detach().cpu().numpy()
|
|
80
|
+
|
|
81
|
+
if hasattr(net, 'W') and net.W is not None:
|
|
82
|
+
W_weights = net.W.detach().cpu().numpy()
|
|
83
|
+
nonlinear_importance = np.linalg.norm(W_weights, axis=1)
|
|
84
|
+
else:
|
|
85
|
+
path_matrix = None
|
|
86
|
+
for layer in net.layers:
|
|
87
|
+
if isinstance(layer, nn.Linear):
|
|
88
|
+
w = np.abs(layer.weight.detach().cpu().numpy())
|
|
89
|
+
if path_matrix is None:
|
|
90
|
+
path_matrix = w[:, :input_dim]
|
|
91
|
+
else:
|
|
92
|
+
if w.shape[1] == path_matrix.shape[0]:
|
|
93
|
+
path_matrix = w @ path_matrix
|
|
94
|
+
else:
|
|
95
|
+
break
|
|
96
|
+
nonlinear_importance = np.linalg.norm(path_matrix, axis=0)
|
|
97
|
+
|
|
98
|
+
return beta_weights, nonlinear_importance
|
|
99
|
+
|
ensurv/_random.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Prediction random-state management."""
|
|
2
|
+
|
|
3
|
+
from contextlib import contextmanager
|
|
4
|
+
import numpy as np
|
|
5
|
+
import torch
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@contextmanager
|
|
9
|
+
def _prediction_rng(seed):
|
|
10
|
+
r"""Temporarily seed prediction draws and restore the saved random states.
|
|
11
|
+
|
|
12
|
+
:param seed: Temporary NumPy and PyTorch seed. If None, use the current random states
|
|
13
|
+
without saving or reseeding them.
|
|
14
|
+
:type seed: int or None
|
|
15
|
+
|
|
16
|
+
:returns: A context in which prediction draws use the requested seed.
|
|
17
|
+
:rtype: context manager
|
|
18
|
+
|
|
19
|
+
With a numeric seed, NumPy, PyTorch CPU, and available CUDA random states are restored even
|
|
20
|
+
if prediction raises an exception.
|
|
21
|
+
"""
|
|
22
|
+
if seed is None:
|
|
23
|
+
yield
|
|
24
|
+
return
|
|
25
|
+
np_state = np.random.get_state()
|
|
26
|
+
cpu_state = torch.get_rng_state()
|
|
27
|
+
cuda_states = torch.cuda.get_rng_state_all() if torch.cuda.is_available() else None
|
|
28
|
+
try:
|
|
29
|
+
np.random.seed(seed)
|
|
30
|
+
torch.manual_seed(seed)
|
|
31
|
+
if cuda_states is not None:
|
|
32
|
+
torch.cuda.manual_seed_all(seed)
|
|
33
|
+
yield
|
|
34
|
+
finally:
|
|
35
|
+
np.random.set_state(np_state)
|
|
36
|
+
torch.set_rng_state(cpu_state)
|
|
37
|
+
if cuda_states is not None:
|
|
38
|
+
torch.cuda.set_rng_state_all(cuda_states)
|
|
39
|
+
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
"""Dataset loading and the three-fold experiment."""
|
|
2
|
+
|
|
3
|
+
import gc
|
|
4
|
+
import os
|
|
5
|
+
import pickle
|
|
6
|
+
import time
|
|
7
|
+
import warnings
|
|
8
|
+
import numpy as np
|
|
9
|
+
import pandas as pd
|
|
10
|
+
import torch
|
|
11
|
+
import matplotlib.pyplot as plt
|
|
12
|
+
from IPython.display import display
|
|
13
|
+
from sklearn.model_selection import StratifiedKFold
|
|
14
|
+
from sklearn.preprocessing import StandardScaler
|
|
15
|
+
from .model import EnSurv
|
|
16
|
+
from .preprocessing import prepare_unos_dataset
|
|
17
|
+
from .evaluation import compute_all_metrics, analyze_hierarchical_importance
|
|
18
|
+
from .utils import plot_time_calibration
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def run_cross_validation(dataset_path='/raid/home/pathak/EnSurv/Kidney_preprocessed_final.csv', output_dir="new_ensurv_cv_outputs"):
|
|
22
|
+
r"""Load the kidney CSV and reproduce the three-fold EnSurv experiment.
|
|
23
|
+
|
|
24
|
+
:param dataset_path: CSV location. The default retains the original notebook absolute path
|
|
25
|
+
to the dataset.
|
|
26
|
+
:type dataset_path: str or path-like, optional
|
|
27
|
+
:param output_dir: Directory for per-fold CSV, NPZ, PDF, state-dictionary, and history
|
|
28
|
+
files; default new_ensurv_cv_outputs. Aggregate CSVs retain their original filenames in
|
|
29
|
+
the current working directory.
|
|
30
|
+
:type output_dir: str or path-like, optional
|
|
31
|
+
|
|
32
|
+
:returns: CV summary tables and the notebook variables needed by the optional post-CV
|
|
33
|
+
plotting examples, including first-fold model/history and last-fold test data.
|
|
34
|
+
:rtype: dict[str, object]
|
|
35
|
+
|
|
36
|
+
Uses StratifiedKFold(n_splits=3, shuffle=True, random_state=42), model
|
|
37
|
+
random_state=42 in every fold, and the original metric prediction seed 42.
|
|
38
|
+
The architecture, training options, external time scaling, domain penalties,
|
|
39
|
+
preprocessing order, and saved output names are preserved. No additional
|
|
40
|
+
deterministic-algorithm settings or random seeds are introduced.
|
|
41
|
+
|
|
42
|
+
Preprocessing occurs before splitting. Each fold fits feature scaling on its
|
|
43
|
+
training rows and divides times by its training maximum. The generator is
|
|
44
|
+
concatenated, with two 512-unit layers, as selected in the notebook. The
|
|
45
|
+
function returns context only after performing the original output writes.
|
|
46
|
+
"""
|
|
47
|
+
warnings.filterwarnings('ignore')
|
|
48
|
+
DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
|
|
49
|
+
|
|
50
|
+
print("="*80)
|
|
51
|
+
print(f"Device: {DEVICE}")
|
|
52
|
+
if torch.cuda.is_available():
|
|
53
|
+
print(f"GPU: {torch.cuda.get_device_name(0)}")
|
|
54
|
+
print(f"GPU Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.2f} GB")
|
|
55
|
+
print("="*80)
|
|
56
|
+
|
|
57
|
+
df = pd.read_csv(dataset_path)
|
|
58
|
+
X, durations, events, feat_names, nominal_cols, center_covariates = prepare_unos_dataset(df)
|
|
59
|
+
|
|
60
|
+
seed = 42
|
|
61
|
+
n_splits = 3
|
|
62
|
+
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)
|
|
63
|
+
fold1_history=None
|
|
64
|
+
fold1_model=None
|
|
65
|
+
|
|
66
|
+
# Where per-fold artifacts go
|
|
67
|
+
OUT_DIR = output_dir
|
|
68
|
+
os.makedirs(OUT_DIR, exist_ok=True)
|
|
69
|
+
|
|
70
|
+
# Storage for cross-validation results
|
|
71
|
+
metrics_list = []
|
|
72
|
+
raw_expl_list = []
|
|
73
|
+
base_feat_expl_list = []
|
|
74
|
+
macro_expl_list = []
|
|
75
|
+
|
|
76
|
+
print(f"Starting {n_splits}-Fold Cross Validation...\n")
|
|
77
|
+
start_total = time.time()
|
|
78
|
+
|
|
79
|
+
for fold, (tr_idx, te_idx) in enumerate(kf.split(X, events)):
|
|
80
|
+
print(f"--- Training Fold {fold + 1}/{n_splits} ---")
|
|
81
|
+
|
|
82
|
+
# 1. Split data (X is a DataFrame, durations/events are numpy arrays)
|
|
83
|
+
X_tr_raw, X_te_raw = X.iloc[tr_idx].copy(), X.iloc[te_idx].copy()
|
|
84
|
+
y_tr, y_te = durations[tr_idx], durations[te_idx]
|
|
85
|
+
e_tr, e_te = events[tr_idx], events[te_idx]
|
|
86
|
+
|
|
87
|
+
# 2. Detect and Scale Features
|
|
88
|
+
binary_cols = [c for c in X.columns if set(X[c].dropna().unique()).issubset({0, 1, 0.0, 1.0})]
|
|
89
|
+
cols_to_scale = [c for c in X.columns if c not in binary_cols and not c.startswith(tuple(nominal_cols))]
|
|
90
|
+
|
|
91
|
+
scaler = StandardScaler()
|
|
92
|
+
X_tr_scaled = X_tr_raw.copy()
|
|
93
|
+
X_te_scaled = X_te_raw.copy()
|
|
94
|
+
|
|
95
|
+
X_tr_scaled[cols_to_scale] = scaler.fit_transform(X_tr_scaled[cols_to_scale]).astype(np.float32)
|
|
96
|
+
X_te_scaled[cols_to_scale] = scaler.transform(X_te_scaled[cols_to_scale]).astype(np.float32)
|
|
97
|
+
|
|
98
|
+
y_max = y_tr.max()
|
|
99
|
+
y_tr_scaled = y_tr / y_max
|
|
100
|
+
|
|
101
|
+
# 3. Initialize and Train Model
|
|
102
|
+
model = EnSurv(
|
|
103
|
+
latent_dim=None, num_layer=2, generator='concatenated', hidden_dims=[512, 512], apply_linear=True,
|
|
104
|
+
dropout=0.1, noise_dist='uniform', noise_std=1.0, batch_normalization=True,
|
|
105
|
+
device=DEVICE, random_state=seed
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
model.fit(
|
|
109
|
+
X_tr_scaled.to_numpy(), y_tr_scaled, e_tr,
|
|
110
|
+
epochs=50, m=5, B=20, verbose=True, patience=10, track_diagnostics=True,
|
|
111
|
+
lambda_energy=0.5, lr=1e-3, km_batch_size=256, batch_size=4096, l1_reg=1e-3, feature_names=X.columns.tolist(),
|
|
112
|
+
domain_constraints={'positive': ['hcvdr_D-R-'], 'negative': ['KDRI', 'CREAT_TRR', 'serum_creatinine', 'center_mean_KDRI', 'first_week_dial']}
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
# 4. Compute Metrics (Unpacking the new returns)
|
|
116
|
+
fold_metrics, t_grid, survival_preds = compute_all_metrics(
|
|
117
|
+
model, y_tr, e_tr, X_te_scaled.to_numpy(), y_te, e_te, y_max=y_max
|
|
118
|
+
)
|
|
119
|
+
metrics_list.append(fold_metrics)
|
|
120
|
+
|
|
121
|
+
# --- save this fold's metrics on their own ---
|
|
122
|
+
pd.DataFrame([fold_metrics]).to_csv(f"{OUT_DIR}/EnSurv_Fold{fold+1}_Metrics.csv", index=False)
|
|
123
|
+
|
|
124
|
+
# Save S_pred and evaluation arrays for offline metric computation
|
|
125
|
+
# Using np.savez_compressed to efficiently bundle arrays into one file
|
|
126
|
+
np.savez_compressed(
|
|
127
|
+
f"{OUT_DIR}/EnSurv_Fold{fold+1}_Predictions.npz",
|
|
128
|
+
S_pred=survival_preds,
|
|
129
|
+
times_grid=t_grid,
|
|
130
|
+
y_te=y_te,
|
|
131
|
+
e_te=e_te,
|
|
132
|
+
y_tr=y_tr,
|
|
133
|
+
e_tr=e_tr
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
# Generate and save calibration plots
|
|
137
|
+
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 6))
|
|
138
|
+
|
|
139
|
+
# Generate plots using the function defined earlier
|
|
140
|
+
plot_time_calibration(survival_preds, t_grid, y_te, e_te, target_time=365.25, ax=ax1)
|
|
141
|
+
plot_time_calibration(survival_preds, t_grid, y_te, e_te, target_time=5*365.25, ax=ax2)
|
|
142
|
+
|
|
143
|
+
plt.tight_layout()
|
|
144
|
+
fig.savefig(f"{OUT_DIR}/EnSurv_Fold{fold+1}_Calibration_Plots.pdf", format='pdf', bbox_inches='tight')
|
|
145
|
+
plt.close(fig) # Prevent memory leaks
|
|
146
|
+
|
|
147
|
+
# 5. Extract Explanations
|
|
148
|
+
raw_explain_df = model.explain(feature_names=X.columns.tolist(), y_max=y_max)
|
|
149
|
+
raw_expl_list.append(raw_explain_df)
|
|
150
|
+
|
|
151
|
+
_, base_feature_importance, macro_level_importance = analyze_hierarchical_importance(
|
|
152
|
+
raw_explain_df, nominal_cols=nominal_cols, center_covariates=center_covariates
|
|
153
|
+
)
|
|
154
|
+
base_feat_expl_list.append(base_feature_importance)
|
|
155
|
+
macro_expl_list.append(macro_level_importance)
|
|
156
|
+
|
|
157
|
+
# --- save this fold's explanations on their own ---
|
|
158
|
+
raw_explain_df.to_csv(f"{OUT_DIR}/EnSurv_Fold{fold+1}_Raw_Explanations.csv", index=False)
|
|
159
|
+
base_feature_importance.to_csv(f"{OUT_DIR}/EnSurv_Fold{fold+1}_BaseFeature_Explanations.csv", index=False)
|
|
160
|
+
macro_level_importance.to_csv(f"{OUT_DIR}/EnSurv_Fold{fold+1}_MacroLevel_Explanations.csv", index=False)
|
|
161
|
+
|
|
162
|
+
# --- save model weights + training diagnostics for this fold ---
|
|
163
|
+
torch.save(model.model.state_dict(), f"{OUT_DIR}/EnSurv_Fold{fold+1}_state_dict.pt")
|
|
164
|
+
with open(f"{OUT_DIR}/EnSurv_Fold{fold+1}_history.pkl", "wb") as f:
|
|
165
|
+
pickle.dump(model.history, f)
|
|
166
|
+
if fold == 0:
|
|
167
|
+
fold1_history = model.history
|
|
168
|
+
fold1_model = model
|
|
169
|
+
|
|
170
|
+
torch.cuda.empty_cache()
|
|
171
|
+
gc.collect()
|
|
172
|
+
|
|
173
|
+
print(f"\nCV Completed in {time.time() - start_total:.2f} seconds.")
|
|
174
|
+
|
|
175
|
+
# ==========================================
|
|
176
|
+
# Post-Processing: Metrics
|
|
177
|
+
# ==========================================
|
|
178
|
+
df_metrics_raw = pd.DataFrame(metrics_list)
|
|
179
|
+
df_metrics_final = pd.DataFrame(index=["EnSurv"])
|
|
180
|
+
|
|
181
|
+
for col in df_metrics_raw.columns:
|
|
182
|
+
col_mean = df_metrics_raw[col].mean()
|
|
183
|
+
col_std = df_metrics_raw[col].std()
|
|
184
|
+
df_metrics_final[col] = rf"${col_mean:.4f}\pm{col_std:.4f}$"
|
|
185
|
+
|
|
186
|
+
print("\nFinal Metrics (Mean ± Std):")
|
|
187
|
+
df_metrics_final.to_csv("EnSurv_3Fold_Results.csv")
|
|
188
|
+
display(df_metrics_final)
|
|
189
|
+
|
|
190
|
+
# ==========================================
|
|
191
|
+
# Post-Processing: Explanations (Averaged across folds)
|
|
192
|
+
# ==========================================
|
|
193
|
+
# 1. Raw Explanations
|
|
194
|
+
df_raw_all = pd.concat(raw_expl_list)
|
|
195
|
+
df_raw_mean = df_raw_all.groupby('Feature').mean().reset_index()
|
|
196
|
+
df_raw_mean = df_raw_mean.sort_values(by='Total_Importance_Pct', ascending=False)
|
|
197
|
+
df_raw_mean.to_csv("512_EnSurv_Raw_Explanations_Mean.csv", index=False)
|
|
198
|
+
|
|
199
|
+
# 2. Base Feature Explanations
|
|
200
|
+
df_base_all = pd.concat(base_feat_expl_list)
|
|
201
|
+
df_base_mean = df_base_all.groupby(['Base_Feature', 'Covariate_Level']).mean().reset_index()
|
|
202
|
+
df_base_mean = df_base_mean.sort_values(by='Total_Importance_Pct', ascending=False)
|
|
203
|
+
df_base_mean.to_csv("512_EnSurv_BaseFeature_Explanations_Mean.csv", index=False)
|
|
204
|
+
|
|
205
|
+
# 3. Macro Level Explanations
|
|
206
|
+
df_macro_all = pd.concat(macro_expl_list)
|
|
207
|
+
df_macro_mean = df_macro_all.groupby('Covariate_Level').mean().reset_index()
|
|
208
|
+
df_macro_mean.to_csv("512_EnSurv_MacroLevel_Explanations_Mean.csv", index=False)
|
|
209
|
+
|
|
210
|
+
return {
|
|
211
|
+
'df_metrics_raw': df_metrics_raw,
|
|
212
|
+
'df_metrics_final': df_metrics_final,
|
|
213
|
+
'df_raw_mean': df_raw_mean,
|
|
214
|
+
'df_base_mean': df_base_mean,
|
|
215
|
+
'df_macro_mean': df_macro_mean,
|
|
216
|
+
'fold1_history': fold1_history,
|
|
217
|
+
'fold1_model': fold1_model,
|
|
218
|
+
'model': model,
|
|
219
|
+
'y_max': y_max,
|
|
220
|
+
'X': X,
|
|
221
|
+
'X_te_scaled': X_te_scaled,
|
|
222
|
+
'X_te_raw': X_te_raw,
|
|
223
|
+
'y_tr': y_tr,
|
|
224
|
+
'y_te': y_te,
|
|
225
|
+
'e_te': e_te,
|
|
226
|
+
'OUT_DIR': OUT_DIR,
|
|
227
|
+
'seed': seed,
|
|
228
|
+
}
|
|
229
|
+
|