ensurv 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ensurv/__init__.py ADDED
@@ -0,0 +1,6 @@
1
+ """EnSurv pooled-completion generative survival model.
2
+ """
3
+
4
+ from .model import EnSurv, EnSurvNet
5
+
6
+ __all__ = ['EnSurv', 'EnSurvNet']
ensurv/_diagnostics.py ADDED
@@ -0,0 +1,99 @@
1
+ """Internal parameter-group and structural-weight diagnostics."""
2
+
3
+ import numpy as np
4
+ import torch.nn as nn
5
+
6
+
7
+ def _get_param_groups(net):
8
+ r"""Collect parameter names into semantic groups for training diagnostics.
9
+
10
+ :param net: Network whose linear pathway, latent projection, hidden layers, normalization
11
+ layers, noise transform, and output layer are inspected.
12
+ :type net: _EnSurvNet
13
+
14
+ :returns: Group labels mapped to names used by ``net.named_parameters()``.
15
+ :rtype: dict[str, list[str]]
16
+ """
17
+ groups = {}
18
+ if net.apply_linear:
19
+ groups['beta (linear)'] = ['beta']
20
+ if net.W is not None:
21
+ groups['W (latent proj)'] = ['W']
22
+ for i in range(net.num_layer):
23
+ keys = [f'layers.{i}.weight', f'layers.{i}.bias']
24
+ if net.bns is not None:
25
+ keys += [f'bns.{i}.weight', f'bns.{i}.bias']
26
+ groups[f'layer {i}'] = keys
27
+ if net.noise_transform is not None:
28
+ groups['noise transform'] = [name for name, _ in net.named_parameters() if name.startswith('noise_transform.')]
29
+ groups['final_layer'] = ['final_layer.weight', 'final_layer.bias']
30
+ return groups
31
+
32
+
33
+ def _group_grad_norms(net, param_groups):
34
+ r"""Compute the root mean square of available gradients in each parameter group.
35
+
36
+ :param net: Network with gradients populated by the current backward pass.
37
+ :type net: _EnSurvNet
38
+ :param param_groups: Semantic group labels and corresponding named parameters, as returned
39
+ by ``_get_param_groups``.
40
+ :type param_groups: dict[str, list[str]]
41
+
42
+ :returns: One RMS gradient magnitude per group; groups without gradients receive zero.
43
+ :rtype: dict[str, float]
44
+
45
+ Call after ``loss.backward()`` and before clearing gradients. For a group with
46
+ :math:`N_g` available gradient entries, the reported value is
47
+ :math:`\{\sum_{j=1}^{N_g} (\nabla_j L)^2 / \max(N_g,1)\}^{1/2}`.
48
+ """
49
+ named = dict(net.named_parameters())
50
+ out = {}
51
+ for gname, keys in param_groups.items():
52
+ sq, cnt = 0.0, 0
53
+ for k in keys:
54
+ p = named.get(k, None)
55
+ if p is not None and p.grad is not None:
56
+ sq += p.grad.detach().pow(2).sum().item()
57
+ cnt += p.grad.numel()
58
+ out[gname] = (sq / max(cnt, 1)) ** 0.5
59
+ return out
60
+
61
+
62
+ def _extract_raw_importance(net, input_dim):
63
+ r"""Extract descriptive linear coefficients and nonlinear structural norms.
64
+
65
+ :param net: Network whose current parameter values are inspected without a forward pass.
66
+ :type net: _EnSurvNet
67
+ :param input_dim: Number of covariates; used to select feature columns from the first
68
+ hidden-layer weight matrix.
69
+ :type input_dim: int
70
+
71
+ :returns: Linear coefficients, or None when disabled, and one nonlinear norm per feature.
72
+ :rtype: tuple[numpy.ndarray or None, numpy.ndarray]
73
+
74
+ With a latent projection, use row norms of its weight matrix. Otherwise multiply compatible
75
+ absolute hidden-layer weight matrices and take feature-column norms.
76
+ """
77
+ beta_weights = None
78
+ if net.apply_linear:
79
+ beta_weights = net.beta.detach().cpu().numpy()
80
+
81
+ if hasattr(net, 'W') and net.W is not None:
82
+ W_weights = net.W.detach().cpu().numpy()
83
+ nonlinear_importance = np.linalg.norm(W_weights, axis=1)
84
+ else:
85
+ path_matrix = None
86
+ for layer in net.layers:
87
+ if isinstance(layer, nn.Linear):
88
+ w = np.abs(layer.weight.detach().cpu().numpy())
89
+ if path_matrix is None:
90
+ path_matrix = w[:, :input_dim]
91
+ else:
92
+ if w.shape[1] == path_matrix.shape[0]:
93
+ path_matrix = w @ path_matrix
94
+ else:
95
+ break
96
+ nonlinear_importance = np.linalg.norm(path_matrix, axis=0)
97
+
98
+ return beta_weights, nonlinear_importance
99
+
ensurv/_random.py ADDED
@@ -0,0 +1,39 @@
1
+ """Prediction random-state management."""
2
+
3
+ from contextlib import contextmanager
4
+ import numpy as np
5
+ import torch
6
+
7
+
8
+ @contextmanager
9
+ def _prediction_rng(seed):
10
+ r"""Temporarily seed prediction draws and restore the saved random states.
11
+
12
+ :param seed: Temporary NumPy and PyTorch seed. If None, use the current random states
13
+ without saving or reseeding them.
14
+ :type seed: int or None
15
+
16
+ :returns: A context in which prediction draws use the requested seed.
17
+ :rtype: context manager
18
+
19
+ With a numeric seed, NumPy, PyTorch CPU, and available CUDA random states are restored even
20
+ if prediction raises an exception.
21
+ """
22
+ if seed is None:
23
+ yield
24
+ return
25
+ np_state = np.random.get_state()
26
+ cpu_state = torch.get_rng_state()
27
+ cuda_states = torch.cuda.get_rng_state_all() if torch.cuda.is_available() else None
28
+ try:
29
+ np.random.seed(seed)
30
+ torch.manual_seed(seed)
31
+ if cuda_states is not None:
32
+ torch.cuda.manual_seed_all(seed)
33
+ yield
34
+ finally:
35
+ np.random.set_state(np_state)
36
+ torch.set_rng_state(cpu_state)
37
+ if cuda_states is not None:
38
+ torch.cuda.set_rng_state_all(cuda_states)
39
+
@@ -0,0 +1,229 @@
1
+ """Dataset loading and the three-fold experiment."""
2
+
3
+ import gc
4
+ import os
5
+ import pickle
6
+ import time
7
+ import warnings
8
+ import numpy as np
9
+ import pandas as pd
10
+ import torch
11
+ import matplotlib.pyplot as plt
12
+ from IPython.display import display
13
+ from sklearn.model_selection import StratifiedKFold
14
+ from sklearn.preprocessing import StandardScaler
15
+ from .model import EnSurv
16
+ from .preprocessing import prepare_unos_dataset
17
+ from .evaluation import compute_all_metrics, analyze_hierarchical_importance
18
+ from .utils import plot_time_calibration
19
+
20
+
21
+ def run_cross_validation(dataset_path='/raid/home/pathak/EnSurv/Kidney_preprocessed_final.csv', output_dir="new_ensurv_cv_outputs"):
22
+ r"""Load the kidney CSV and reproduce the three-fold EnSurv experiment.
23
+
24
+ :param dataset_path: CSV location. The default retains the original notebook absolute path
25
+ to the dataset.
26
+ :type dataset_path: str or path-like, optional
27
+ :param output_dir: Directory for per-fold CSV, NPZ, PDF, state-dictionary, and history
28
+ files; default new_ensurv_cv_outputs. Aggregate CSVs retain their original filenames in
29
+ the current working directory.
30
+ :type output_dir: str or path-like, optional
31
+
32
+ :returns: CV summary tables and the notebook variables needed by the optional post-CV
33
+ plotting examples, including first-fold model/history and last-fold test data.
34
+ :rtype: dict[str, object]
35
+
36
+ Uses StratifiedKFold(n_splits=3, shuffle=True, random_state=42), model
37
+ random_state=42 in every fold, and the original metric prediction seed 42.
38
+ The architecture, training options, external time scaling, domain penalties,
39
+ preprocessing order, and saved output names are preserved. No additional
40
+ deterministic-algorithm settings or random seeds are introduced.
41
+
42
+ Preprocessing occurs before splitting. Each fold fits feature scaling on its
43
+ training rows and divides times by its training maximum. The generator is
44
+ concatenated, with two 512-unit layers, as selected in the notebook. The
45
+ function returns context only after performing the original output writes.
46
+ """
47
+ warnings.filterwarnings('ignore')
48
+ DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
49
+
50
+ print("="*80)
51
+ print(f"Device: {DEVICE}")
52
+ if torch.cuda.is_available():
53
+ print(f"GPU: {torch.cuda.get_device_name(0)}")
54
+ print(f"GPU Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.2f} GB")
55
+ print("="*80)
56
+
57
+ df = pd.read_csv(dataset_path)
58
+ X, durations, events, feat_names, nominal_cols, center_covariates = prepare_unos_dataset(df)
59
+
60
+ seed = 42
61
+ n_splits = 3
62
+ kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)
63
+ fold1_history=None
64
+ fold1_model=None
65
+
66
+ # Where per-fold artifacts go
67
+ OUT_DIR = output_dir
68
+ os.makedirs(OUT_DIR, exist_ok=True)
69
+
70
+ # Storage for cross-validation results
71
+ metrics_list = []
72
+ raw_expl_list = []
73
+ base_feat_expl_list = []
74
+ macro_expl_list = []
75
+
76
+ print(f"Starting {n_splits}-Fold Cross Validation...\n")
77
+ start_total = time.time()
78
+
79
+ for fold, (tr_idx, te_idx) in enumerate(kf.split(X, events)):
80
+ print(f"--- Training Fold {fold + 1}/{n_splits} ---")
81
+
82
+ # 1. Split data (X is a DataFrame, durations/events are numpy arrays)
83
+ X_tr_raw, X_te_raw = X.iloc[tr_idx].copy(), X.iloc[te_idx].copy()
84
+ y_tr, y_te = durations[tr_idx], durations[te_idx]
85
+ e_tr, e_te = events[tr_idx], events[te_idx]
86
+
87
+ # 2. Detect and Scale Features
88
+ binary_cols = [c for c in X.columns if set(X[c].dropna().unique()).issubset({0, 1, 0.0, 1.0})]
89
+ cols_to_scale = [c for c in X.columns if c not in binary_cols and not c.startswith(tuple(nominal_cols))]
90
+
91
+ scaler = StandardScaler()
92
+ X_tr_scaled = X_tr_raw.copy()
93
+ X_te_scaled = X_te_raw.copy()
94
+
95
+ X_tr_scaled[cols_to_scale] = scaler.fit_transform(X_tr_scaled[cols_to_scale]).astype(np.float32)
96
+ X_te_scaled[cols_to_scale] = scaler.transform(X_te_scaled[cols_to_scale]).astype(np.float32)
97
+
98
+ y_max = y_tr.max()
99
+ y_tr_scaled = y_tr / y_max
100
+
101
+ # 3. Initialize and Train Model
102
+ model = EnSurv(
103
+ latent_dim=None, num_layer=2, generator='concatenated', hidden_dims=[512, 512], apply_linear=True,
104
+ dropout=0.1, noise_dist='uniform', noise_std=1.0, batch_normalization=True,
105
+ device=DEVICE, random_state=seed
106
+ )
107
+
108
+ model.fit(
109
+ X_tr_scaled.to_numpy(), y_tr_scaled, e_tr,
110
+ epochs=50, m=5, B=20, verbose=True, patience=10, track_diagnostics=True,
111
+ lambda_energy=0.5, lr=1e-3, km_batch_size=256, batch_size=4096, l1_reg=1e-3, feature_names=X.columns.tolist(),
112
+ domain_constraints={'positive': ['hcvdr_D-R-'], 'negative': ['KDRI', 'CREAT_TRR', 'serum_creatinine', 'center_mean_KDRI', 'first_week_dial']}
113
+ )
114
+
115
+ # 4. Compute Metrics (Unpacking the new returns)
116
+ fold_metrics, t_grid, survival_preds = compute_all_metrics(
117
+ model, y_tr, e_tr, X_te_scaled.to_numpy(), y_te, e_te, y_max=y_max
118
+ )
119
+ metrics_list.append(fold_metrics)
120
+
121
+ # --- save this fold's metrics on their own ---
122
+ pd.DataFrame([fold_metrics]).to_csv(f"{OUT_DIR}/EnSurv_Fold{fold+1}_Metrics.csv", index=False)
123
+
124
+ # Save S_pred and evaluation arrays for offline metric computation
125
+ # Using np.savez_compressed to efficiently bundle arrays into one file
126
+ np.savez_compressed(
127
+ f"{OUT_DIR}/EnSurv_Fold{fold+1}_Predictions.npz",
128
+ S_pred=survival_preds,
129
+ times_grid=t_grid,
130
+ y_te=y_te,
131
+ e_te=e_te,
132
+ y_tr=y_tr,
133
+ e_tr=e_tr
134
+ )
135
+
136
+ # Generate and save calibration plots
137
+ fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 6))
138
+
139
+ # Generate plots using the function defined earlier
140
+ plot_time_calibration(survival_preds, t_grid, y_te, e_te, target_time=365.25, ax=ax1)
141
+ plot_time_calibration(survival_preds, t_grid, y_te, e_te, target_time=5*365.25, ax=ax2)
142
+
143
+ plt.tight_layout()
144
+ fig.savefig(f"{OUT_DIR}/EnSurv_Fold{fold+1}_Calibration_Plots.pdf", format='pdf', bbox_inches='tight')
145
+ plt.close(fig) # Prevent memory leaks
146
+
147
+ # 5. Extract Explanations
148
+ raw_explain_df = model.explain(feature_names=X.columns.tolist(), y_max=y_max)
149
+ raw_expl_list.append(raw_explain_df)
150
+
151
+ _, base_feature_importance, macro_level_importance = analyze_hierarchical_importance(
152
+ raw_explain_df, nominal_cols=nominal_cols, center_covariates=center_covariates
153
+ )
154
+ base_feat_expl_list.append(base_feature_importance)
155
+ macro_expl_list.append(macro_level_importance)
156
+
157
+ # --- save this fold's explanations on their own ---
158
+ raw_explain_df.to_csv(f"{OUT_DIR}/EnSurv_Fold{fold+1}_Raw_Explanations.csv", index=False)
159
+ base_feature_importance.to_csv(f"{OUT_DIR}/EnSurv_Fold{fold+1}_BaseFeature_Explanations.csv", index=False)
160
+ macro_level_importance.to_csv(f"{OUT_DIR}/EnSurv_Fold{fold+1}_MacroLevel_Explanations.csv", index=False)
161
+
162
+ # --- save model weights + training diagnostics for this fold ---
163
+ torch.save(model.model.state_dict(), f"{OUT_DIR}/EnSurv_Fold{fold+1}_state_dict.pt")
164
+ with open(f"{OUT_DIR}/EnSurv_Fold{fold+1}_history.pkl", "wb") as f:
165
+ pickle.dump(model.history, f)
166
+ if fold == 0:
167
+ fold1_history = model.history
168
+ fold1_model = model
169
+
170
+ torch.cuda.empty_cache()
171
+ gc.collect()
172
+
173
+ print(f"\nCV Completed in {time.time() - start_total:.2f} seconds.")
174
+
175
+ # ==========================================
176
+ # Post-Processing: Metrics
177
+ # ==========================================
178
+ df_metrics_raw = pd.DataFrame(metrics_list)
179
+ df_metrics_final = pd.DataFrame(index=["EnSurv"])
180
+
181
+ for col in df_metrics_raw.columns:
182
+ col_mean = df_metrics_raw[col].mean()
183
+ col_std = df_metrics_raw[col].std()
184
+ df_metrics_final[col] = rf"${col_mean:.4f}\pm{col_std:.4f}$"
185
+
186
+ print("\nFinal Metrics (Mean ± Std):")
187
+ df_metrics_final.to_csv("EnSurv_3Fold_Results.csv")
188
+ display(df_metrics_final)
189
+
190
+ # ==========================================
191
+ # Post-Processing: Explanations (Averaged across folds)
192
+ # ==========================================
193
+ # 1. Raw Explanations
194
+ df_raw_all = pd.concat(raw_expl_list)
195
+ df_raw_mean = df_raw_all.groupby('Feature').mean().reset_index()
196
+ df_raw_mean = df_raw_mean.sort_values(by='Total_Importance_Pct', ascending=False)
197
+ df_raw_mean.to_csv("512_EnSurv_Raw_Explanations_Mean.csv", index=False)
198
+
199
+ # 2. Base Feature Explanations
200
+ df_base_all = pd.concat(base_feat_expl_list)
201
+ df_base_mean = df_base_all.groupby(['Base_Feature', 'Covariate_Level']).mean().reset_index()
202
+ df_base_mean = df_base_mean.sort_values(by='Total_Importance_Pct', ascending=False)
203
+ df_base_mean.to_csv("512_EnSurv_BaseFeature_Explanations_Mean.csv", index=False)
204
+
205
+ # 3. Macro Level Explanations
206
+ df_macro_all = pd.concat(macro_expl_list)
207
+ df_macro_mean = df_macro_all.groupby('Covariate_Level').mean().reset_index()
208
+ df_macro_mean.to_csv("512_EnSurv_MacroLevel_Explanations_Mean.csv", index=False)
209
+
210
+ return {
211
+ 'df_metrics_raw': df_metrics_raw,
212
+ 'df_metrics_final': df_metrics_final,
213
+ 'df_raw_mean': df_raw_mean,
214
+ 'df_base_mean': df_base_mean,
215
+ 'df_macro_mean': df_macro_mean,
216
+ 'fold1_history': fold1_history,
217
+ 'fold1_model': fold1_model,
218
+ 'model': model,
219
+ 'y_max': y_max,
220
+ 'X': X,
221
+ 'X_te_scaled': X_te_scaled,
222
+ 'X_te_raw': X_te_raw,
223
+ 'y_tr': y_tr,
224
+ 'y_te': y_te,
225
+ 'e_te': e_te,
226
+ 'OUT_DIR': OUT_DIR,
227
+ 'seed': seed,
228
+ }
229
+