combatlearn 1.2.0__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {combatlearn-1.2.0 → combatlearn-1.4.0}/PKG-INFO +7 -3
- {combatlearn-1.2.0 → combatlearn-1.4.0}/README.md +1 -1
- {combatlearn-1.2.0 → combatlearn-1.4.0}/combatlearn/__init__.py +1 -1
- {combatlearn-1.2.0 → combatlearn-1.4.0}/combatlearn/core.py +282 -85
- {combatlearn-1.2.0 → combatlearn-1.4.0}/combatlearn/metrics.py +186 -127
- combatlearn-1.4.0/combatlearn/py.typed +0 -0
- combatlearn-1.4.0/combatlearn/sklearn_api.py +413 -0
- {combatlearn-1.2.0 → combatlearn-1.4.0}/combatlearn/visualization.py +227 -151
- {combatlearn-1.2.0 → combatlearn-1.4.0}/combatlearn.egg-info/PKG-INFO +7 -3
- {combatlearn-1.2.0 → combatlearn-1.4.0}/combatlearn.egg-info/SOURCES.txt +3 -2
- {combatlearn-1.2.0 → combatlearn-1.4.0}/combatlearn.egg-info/requires.txt +5 -1
- {combatlearn-1.2.0 → combatlearn-1.4.0}/pyproject.toml +19 -3
- combatlearn-1.4.0/requirements-dev.txt +6 -0
- combatlearn-1.2.0/docs_requirements.txt → combatlearn-1.4.0/requirements-docs.txt +1 -1
- {combatlearn-1.2.0 → combatlearn-1.4.0}/requirements.txt +1 -0
- {combatlearn-1.2.0 → combatlearn-1.4.0}/tests/test_combat.py +358 -5
- combatlearn-1.2.0/combatlearn/sklearn_api.py +0 -143
- combatlearn-1.2.0/dev_requirements.txt +0 -3
- {combatlearn-1.2.0 → combatlearn-1.4.0}/LICENSE +0 -0
- {combatlearn-1.2.0 → combatlearn-1.4.0}/combatlearn.egg-info/dependency_links.txt +0 -0
- {combatlearn-1.2.0 → combatlearn-1.4.0}/combatlearn.egg-info/top_level.txt +0 -0
- {combatlearn-1.2.0 → combatlearn-1.4.0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: combatlearn
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: Batch-effect harmonization for machine learning frameworks.
|
|
5
5
|
Author-email: Ettore Rocchi <ettoreroc@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -16,17 +16,21 @@ Requires-Dist: pandas>=1.3
|
|
|
16
16
|
Requires-Dist: numpy>=1.21
|
|
17
17
|
Requires-Dist: scikit-learn>=1.2
|
|
18
18
|
Requires-Dist: matplotlib>=3.4
|
|
19
|
+
Requires-Dist: seaborn>=0.12
|
|
19
20
|
Requires-Dist: plotly>=5.0
|
|
20
21
|
Requires-Dist: nbformat>=4.2
|
|
21
22
|
Requires-Dist: umap-learn>=0.5
|
|
22
23
|
Provides-Extra: dev
|
|
23
24
|
Requires-Dist: pytest>=7; extra == "dev"
|
|
25
|
+
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
24
26
|
Requires-Dist: ruff>=0.8; extra == "dev"
|
|
25
27
|
Requires-Dist: pre-commit>=3.0; extra == "dev"
|
|
28
|
+
Requires-Dist: mypy>=1.0; extra == "dev"
|
|
29
|
+
Requires-Dist: pandas-stubs; extra == "dev"
|
|
26
30
|
Provides-Extra: docs
|
|
27
31
|
Requires-Dist: sphinx>=7.0.0; extra == "docs"
|
|
28
32
|
Requires-Dist: pydata-sphinx-theme>=0.15.0; extra == "docs"
|
|
29
|
-
Requires-Dist: myst-
|
|
33
|
+
Requires-Dist: myst-nb>=1.1.0; extra == "docs"
|
|
30
34
|
Requires-Dist: sphinx-copybutton>=0.5.0; extra == "docs"
|
|
31
35
|
Requires-Dist: linkify-it-py>=2.0.0; extra == "docs"
|
|
32
36
|
Dynamic: license-file
|
|
@@ -76,7 +80,7 @@ from combatlearn import ComBat
|
|
|
76
80
|
df = pd.read_csv("data.csv", index_col=0)
|
|
77
81
|
X, y = df.drop(columns="y"), df["y"]
|
|
78
82
|
|
|
79
|
-
batch = pd.read_csv("batch.csv", index_col=0
|
|
83
|
+
batch = pd.read_csv("batch.csv", index_col=0).squeeze("columns")
|
|
80
84
|
diag = pd.read_csv("diagnosis.csv", index_col=0) # categorical
|
|
81
85
|
age = pd.read_csv("age.csv", index_col=0) # continuous
|
|
82
86
|
|
|
@@ -43,7 +43,7 @@ from combatlearn import ComBat
|
|
|
43
43
|
df = pd.read_csv("data.csv", index_col=0)
|
|
44
44
|
X, y = df.drop(columns="y"), df["y"]
|
|
45
45
|
|
|
46
|
-
batch = pd.read_csv("batch.csv", index_col=0
|
|
46
|
+
batch = pd.read_csv("batch.csv", index_col=0).squeeze("columns")
|
|
47
47
|
diag = pd.read_csv("diagnosis.csv", index_col=0) # categorical
|
|
48
48
|
age = pd.read_csv("age.csv", index_col=0) # continuous
|
|
49
49
|
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
import warnings
|
|
12
|
-
from typing import Any, Literal
|
|
12
|
+
from typing import Any, Literal, TypeAlias
|
|
13
13
|
|
|
14
14
|
import numpy as np
|
|
15
15
|
import numpy.linalg as la
|
|
@@ -17,8 +17,8 @@ import numpy.typing as npt
|
|
|
17
17
|
import pandas as pd
|
|
18
18
|
from sklearn.decomposition import PCA
|
|
19
19
|
|
|
20
|
-
ArrayLike = pd.DataFrame | pd.Series | npt.NDArray[Any]
|
|
21
|
-
FloatArray = npt.NDArray[np.float64]
|
|
20
|
+
ArrayLike: TypeAlias = pd.DataFrame | pd.Series | npt.NDArray[Any]
|
|
21
|
+
FloatArray: TypeAlias = npt.NDArray[np.float64]
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
class ComBatModel:
|
|
@@ -69,6 +69,7 @@ class ComBatModel:
|
|
|
69
69
|
self._n_per_batch: dict[str, int]
|
|
70
70
|
self._reference_batch_idx: int | None
|
|
71
71
|
self._beta_hat_nonbatch: FloatArray
|
|
72
|
+
self._nonbatch_columns: list[str]
|
|
72
73
|
self._n_batch: int
|
|
73
74
|
self._p_design: int
|
|
74
75
|
self._covbat_pca: PCA
|
|
@@ -80,19 +81,35 @@ class ComBatModel:
|
|
|
80
81
|
# Validate covbat_cov_thresh
|
|
81
82
|
if isinstance(self.covbat_cov_thresh, float):
|
|
82
83
|
if not (0.0 < self.covbat_cov_thresh <= 1.0):
|
|
83
|
-
raise ValueError(
|
|
84
|
+
raise ValueError(
|
|
85
|
+
f"covbat_cov_thresh={self.covbat_cov_thresh!r} is out of range. "
|
|
86
|
+
f"Expected a float in (0, 1] (e.g. 0.9 to retain 90% of variance)."
|
|
87
|
+
)
|
|
84
88
|
elif isinstance(self.covbat_cov_thresh, int):
|
|
85
89
|
if self.covbat_cov_thresh < 1:
|
|
86
|
-
raise ValueError(
|
|
90
|
+
raise ValueError(
|
|
91
|
+
f"covbat_cov_thresh={self.covbat_cov_thresh!r} is invalid. "
|
|
92
|
+
f"Expected an int >= 1 (the number of principal components to retain)."
|
|
93
|
+
)
|
|
87
94
|
else:
|
|
88
|
-
raise TypeError(
|
|
95
|
+
raise TypeError(
|
|
96
|
+
f"covbat_cov_thresh must be float or int, "
|
|
97
|
+
f"got {type(self.covbat_cov_thresh).__name__} ({self.covbat_cov_thresh!r}). "
|
|
98
|
+
f"Use a float in (0, 1] for cumulative variance or an int >= 1 for "
|
|
99
|
+
f"a fixed number of PCs."
|
|
100
|
+
)
|
|
89
101
|
|
|
90
102
|
@staticmethod
|
|
91
103
|
def _as_series(arr: ArrayLike, index: pd.Index, name: str) -> pd.Series:
|
|
92
104
|
"""Convert array-like to categorical Series with validation."""
|
|
93
105
|
ser = arr.copy() if isinstance(arr, pd.Series) else pd.Series(arr, index=index, name=name)
|
|
94
106
|
if not ser.index.equals(index):
|
|
95
|
-
|
|
107
|
+
diff = index.symmetric_difference(ser.index)
|
|
108
|
+
raise ValueError(
|
|
109
|
+
f"`{name}` index does not align with `X`. "
|
|
110
|
+
f"{len(diff)} mismatched entries (first 3): {list(diff[:3])}. "
|
|
111
|
+
f"Ensure both share the same index."
|
|
112
|
+
)
|
|
96
113
|
return ser.astype("category")
|
|
97
114
|
|
|
98
115
|
@staticmethod
|
|
@@ -105,7 +122,12 @@ class ComBatModel:
|
|
|
105
122
|
if not isinstance(arr, pd.DataFrame):
|
|
106
123
|
arr = pd.DataFrame(arr, index=index)
|
|
107
124
|
if not arr.index.equals(index):
|
|
108
|
-
|
|
125
|
+
diff = index.symmetric_difference(arr.index)
|
|
126
|
+
raise ValueError(
|
|
127
|
+
f"`{name}` index does not align with `X`. "
|
|
128
|
+
f"{len(diff)} mismatched entries (first 3): {list(diff[:3])}. "
|
|
129
|
+
f"Ensure both share the same index."
|
|
130
|
+
)
|
|
109
131
|
return arr
|
|
110
132
|
|
|
111
133
|
def fit(
|
|
@@ -117,10 +139,32 @@ class ComBatModel:
|
|
|
117
139
|
discrete_covariates: ArrayLike | None = None,
|
|
118
140
|
continuous_covariates: ArrayLike | None = None,
|
|
119
141
|
) -> ComBatModel:
|
|
120
|
-
"""Fit the ComBat model.
|
|
142
|
+
"""Fit the ComBat model.
|
|
143
|
+
|
|
144
|
+
Parameters
|
|
145
|
+
----------
|
|
146
|
+
X : array-like of shape (n_samples, n_features)
|
|
147
|
+
Input data to fit.
|
|
148
|
+
y : None
|
|
149
|
+
Ignored. Present for API compatibility.
|
|
150
|
+
batch : array-like of shape (n_samples,)
|
|
151
|
+
Batch labels for each sample.
|
|
152
|
+
discrete_covariates : array-like or None, default=None
|
|
153
|
+
Categorical covariates to protect (Fortin/Chen only).
|
|
154
|
+
continuous_covariates : array-like or None, default=None
|
|
155
|
+
Continuous covariates to protect (Fortin/Chen only).
|
|
156
|
+
|
|
157
|
+
Returns
|
|
158
|
+
-------
|
|
159
|
+
self : ComBatModel
|
|
160
|
+
Fitted model.
|
|
161
|
+
"""
|
|
121
162
|
method = self.method.lower()
|
|
122
163
|
if method not in {"johnson", "fortin", "chen"}:
|
|
123
|
-
raise ValueError(
|
|
164
|
+
raise ValueError(
|
|
165
|
+
f"method={self.method!r} is not recognized. "
|
|
166
|
+
f"Expected one of 'johnson', 'fortin', or 'chen'."
|
|
167
|
+
)
|
|
124
168
|
if not isinstance(X, pd.DataFrame):
|
|
125
169
|
X = pd.DataFrame(X)
|
|
126
170
|
idx = X.index
|
|
@@ -131,8 +175,9 @@ class ComBatModel:
|
|
|
131
175
|
|
|
132
176
|
if self.reference_batch is not None and self.reference_batch not in batch.cat.categories:
|
|
133
177
|
raise ValueError(
|
|
134
|
-
f"reference_batch={self.reference_batch!r} not
|
|
135
|
-
f"{list(batch.cat.categories)}"
|
|
178
|
+
f"reference_batch={self.reference_batch!r} not found. "
|
|
179
|
+
f"Available batches: {list(batch.cat.categories)}. "
|
|
180
|
+
f"Check for typos or trailing whitespace in batch labels."
|
|
136
181
|
)
|
|
137
182
|
|
|
138
183
|
if method == "johnson":
|
|
@@ -148,9 +193,16 @@ class ComBatModel:
|
|
|
148
193
|
def _fit_johnson(self, X: pd.DataFrame, batch: pd.Series) -> None:
|
|
149
194
|
"""Johnson et al. (2007) ComBat."""
|
|
150
195
|
self._batch_levels = batch.cat.categories
|
|
151
|
-
pooled_var = X.var(axis=0, ddof=1) + self.eps
|
|
152
196
|
grand_mean = X.mean(axis=0)
|
|
153
197
|
|
|
198
|
+
# Within-batch residual variance
|
|
199
|
+
n_samples = len(X)
|
|
200
|
+
resid = X.copy()
|
|
201
|
+
for lvl in self._batch_levels:
|
|
202
|
+
idx = batch == lvl
|
|
203
|
+
resid.loc[idx] = X.loc[idx] - X.loc[idx].mean(axis=0)
|
|
204
|
+
pooled_var = (resid**2).sum(axis=0) / n_samples + self.eps
|
|
205
|
+
|
|
154
206
|
Xs = (X - grand_mean) / np.sqrt(pooled_var)
|
|
155
207
|
|
|
156
208
|
n_per_batch: dict[str, int] = {}
|
|
@@ -161,7 +213,10 @@ class ComBatModel:
|
|
|
161
213
|
idx = batch == lvl
|
|
162
214
|
n_b = int(idx.sum())
|
|
163
215
|
if n_b < 2:
|
|
164
|
-
raise ValueError(
|
|
216
|
+
raise ValueError(
|
|
217
|
+
f"Batch '{lvl}' has only {n_b} sample(s). ComBat requires >= 2 samples "
|
|
218
|
+
f"per batch. Consider merging small batches or removing them."
|
|
219
|
+
)
|
|
165
220
|
n_per_batch[str(lvl)] = n_b
|
|
166
221
|
xb = Xs.loc[idx]
|
|
167
222
|
gamma_hat.append(xb.mean(axis=0).values)
|
|
@@ -170,14 +225,23 @@ class ComBatModel:
|
|
|
170
225
|
gamma_hat_arr = np.vstack(gamma_hat)
|
|
171
226
|
delta_hat_arr = np.vstack(delta_hat)
|
|
172
227
|
|
|
228
|
+
b_names = [str(lvl) for lvl in self._batch_levels]
|
|
173
229
|
if self.mean_only:
|
|
174
230
|
gamma_star = self._shrink_gamma(
|
|
175
|
-
gamma_hat_arr,
|
|
231
|
+
gamma_hat_arr,
|
|
232
|
+
delta_hat_arr,
|
|
233
|
+
n_per_batch,
|
|
234
|
+
parametric=self.parametric,
|
|
235
|
+
batch_names=b_names,
|
|
176
236
|
)
|
|
177
237
|
delta_star = np.ones_like(delta_hat_arr)
|
|
178
238
|
else:
|
|
179
239
|
gamma_star, delta_star = self._shrink_gamma_delta(
|
|
180
|
-
gamma_hat_arr,
|
|
240
|
+
gamma_hat_arr,
|
|
241
|
+
delta_hat_arr,
|
|
242
|
+
n_per_batch,
|
|
243
|
+
parametric=self.parametric,
|
|
244
|
+
batch_names=b_names,
|
|
181
245
|
)
|
|
182
246
|
|
|
183
247
|
if self.reference_batch is not None:
|
|
@@ -196,6 +260,7 @@ class ComBatModel:
|
|
|
196
260
|
self._gamma_star = gamma_star
|
|
197
261
|
self._delta_star = delta_star
|
|
198
262
|
self._n_per_batch = n_per_batch
|
|
263
|
+
self._design_cond: float | None = None
|
|
199
264
|
|
|
200
265
|
def _fit_fortin(
|
|
201
266
|
self,
|
|
@@ -213,8 +278,8 @@ class ComBatModel:
|
|
|
213
278
|
if self.reference_batch is not None:
|
|
214
279
|
if self.reference_batch not in self._batch_levels:
|
|
215
280
|
raise ValueError(
|
|
216
|
-
f"reference_batch={self.reference_batch!r} not
|
|
217
|
-
f"{list(self._batch_levels)}"
|
|
281
|
+
f"reference_batch={self.reference_batch!r} not found. "
|
|
282
|
+
f"Available batches: {list(self._batch_levels)}. "
|
|
218
283
|
)
|
|
219
284
|
batch_dummies.loc[:, self.reference_batch] = 1.0
|
|
220
285
|
|
|
@@ -225,7 +290,10 @@ class ComBatModel:
|
|
|
225
290
|
if cont is not None:
|
|
226
291
|
parts.append(cont.astype(float))
|
|
227
292
|
|
|
228
|
-
|
|
293
|
+
design_df = pd.concat(parts, axis=1)
|
|
294
|
+
self._nonbatch_columns = design_df.columns[n_batch:].tolist()
|
|
295
|
+
design = design_df.values
|
|
296
|
+
self._design_cond = float(la.cond(design))
|
|
229
297
|
p_design = design.shape[1]
|
|
230
298
|
|
|
231
299
|
X_np = X.values
|
|
@@ -234,8 +302,10 @@ class ComBatModel:
|
|
|
234
302
|
beta_hat_batch = beta_hat[:n_batch]
|
|
235
303
|
self._beta_hat_nonbatch = beta_hat[n_batch:]
|
|
236
304
|
|
|
237
|
-
n_per_batch =
|
|
238
|
-
|
|
305
|
+
n_per_batch: FloatArray = np.asarray(
|
|
306
|
+
batch.value_counts().sort_index().astype(int).values, dtype=np.float64
|
|
307
|
+
)
|
|
308
|
+
self._n_per_batch = dict(zip(self._batch_levels, n_per_batch.astype(int), strict=True))
|
|
239
309
|
|
|
240
310
|
if self.reference_batch is not None:
|
|
241
311
|
ref_idx = list(self._batch_levels).index(self.reference_batch)
|
|
@@ -247,7 +317,7 @@ class ComBatModel:
|
|
|
247
317
|
self._grand_mean = pd.Series(grand_mean, index=X.columns)
|
|
248
318
|
|
|
249
319
|
if self.reference_batch is not None:
|
|
250
|
-
ref_mask = (batch == self.reference_batch)
|
|
320
|
+
ref_mask = np.asarray(batch == self.reference_batch)
|
|
251
321
|
resid = X_np[ref_mask] - design[ref_mask] @ beta_hat
|
|
252
322
|
denom = int(ref_mask.sum())
|
|
253
323
|
else:
|
|
@@ -264,14 +334,23 @@ class ComBatModel:
|
|
|
264
334
|
[Xs[batch == lvl].var(axis=0, ddof=1) + self.eps for lvl in self._batch_levels]
|
|
265
335
|
)
|
|
266
336
|
|
|
337
|
+
b_names = [str(lvl) for lvl in self._batch_levels]
|
|
267
338
|
if self.mean_only:
|
|
268
339
|
gamma_star = self._shrink_gamma(
|
|
269
|
-
gamma_hat,
|
|
340
|
+
gamma_hat,
|
|
341
|
+
delta_hat,
|
|
342
|
+
n_per_batch,
|
|
343
|
+
parametric=self.parametric,
|
|
344
|
+
batch_names=b_names,
|
|
270
345
|
)
|
|
271
346
|
delta_star = np.ones_like(delta_hat)
|
|
272
347
|
else:
|
|
273
348
|
gamma_star, delta_star = self._shrink_gamma_delta(
|
|
274
|
-
gamma_hat,
|
|
349
|
+
gamma_hat,
|
|
350
|
+
delta_hat,
|
|
351
|
+
n_per_batch,
|
|
352
|
+
parametric=self.parametric,
|
|
353
|
+
batch_names=b_names,
|
|
275
354
|
)
|
|
276
355
|
|
|
277
356
|
if ref_idx is not None:
|
|
@@ -322,14 +401,23 @@ class ComBatModel:
|
|
|
322
401
|
gamma_hat_arr = np.vstack(gamma_hat)
|
|
323
402
|
delta_hat_arr = np.vstack(delta_hat)
|
|
324
403
|
|
|
404
|
+
b_names = [str(lvl) for lvl in self._batch_levels_pc]
|
|
325
405
|
if self.mean_only:
|
|
326
406
|
gamma_star = self._shrink_gamma(
|
|
327
|
-
gamma_hat_arr,
|
|
407
|
+
gamma_hat_arr,
|
|
408
|
+
delta_hat_arr,
|
|
409
|
+
n_per_batch,
|
|
410
|
+
parametric=self.parametric,
|
|
411
|
+
batch_names=b_names,
|
|
328
412
|
)
|
|
329
413
|
delta_star = np.ones_like(delta_hat_arr)
|
|
330
414
|
else:
|
|
331
415
|
gamma_star, delta_star = self._shrink_gamma_delta(
|
|
332
|
-
gamma_hat_arr,
|
|
416
|
+
gamma_hat_arr,
|
|
417
|
+
delta_hat_arr,
|
|
418
|
+
n_per_batch,
|
|
419
|
+
parametric=self.parametric,
|
|
420
|
+
batch_names=b_names,
|
|
333
421
|
)
|
|
334
422
|
|
|
335
423
|
if self.reference_batch is not None:
|
|
@@ -352,64 +440,105 @@ class ComBatModel:
|
|
|
352
440
|
parametric: bool,
|
|
353
441
|
max_iter: int = 100,
|
|
354
442
|
tol: float = 1e-4,
|
|
443
|
+
batch_names: list[str] | None = None,
|
|
355
444
|
) -> tuple[FloatArray, FloatArray]:
|
|
356
|
-
"""Empirical Bayes shrinkage estimation.
|
|
445
|
+
"""Empirical Bayes shrinkage estimation.
|
|
446
|
+
|
|
447
|
+
Both the parametric and non-parametric branches use an iterative
|
|
448
|
+
scheme that alternates between updating gamma (location) and delta
|
|
449
|
+
(scale) posteriors until convergence.
|
|
450
|
+
"""
|
|
451
|
+
B, _p = gamma_hat.shape
|
|
452
|
+
n_vec = (
|
|
453
|
+
np.array(list(n_per_batch.values())) if isinstance(n_per_batch, dict) else n_per_batch
|
|
454
|
+
)
|
|
455
|
+
gamma_bar = gamma_hat.mean(axis=0)
|
|
456
|
+
t2 = np.maximum(gamma_hat.var(axis=0, ddof=1), self.eps)
|
|
457
|
+
convergence_info: list[dict[str, object]] = []
|
|
458
|
+
|
|
459
|
+
def postmean(
|
|
460
|
+
g_hat: FloatArray,
|
|
461
|
+
g_bar: FloatArray,
|
|
462
|
+
n: float,
|
|
463
|
+
d_star: FloatArray,
|
|
464
|
+
t2_: FloatArray,
|
|
465
|
+
) -> FloatArray:
|
|
466
|
+
return (t2_ * n * g_hat + d_star * g_bar) / (t2_ * n + d_star)
|
|
467
|
+
|
|
468
|
+
def postvar(
|
|
469
|
+
sum2: FloatArray, n: float, a: FloatArray | float, b: FloatArray | float
|
|
470
|
+
) -> FloatArray:
|
|
471
|
+
return (0.5 * sum2 + b) / (n / 2.0 + a - 1.0)
|
|
472
|
+
|
|
357
473
|
if parametric:
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
a_prior = (delta_hat.mean(axis=0) ** 2) / delta_hat.var(axis=0, ddof=1) + 2
|
|
474
|
+
delta_var = np.maximum(delta_hat.var(axis=0, ddof=1), self.eps)
|
|
475
|
+
a_prior = (delta_hat.mean(axis=0) ** 2) / delta_var + 2
|
|
361
476
|
b_prior = delta_hat.mean(axis=0) * (a_prior - 1)
|
|
362
477
|
|
|
363
|
-
B, _p = gamma_hat.shape
|
|
364
478
|
gamma_star = np.empty_like(gamma_hat)
|
|
365
479
|
delta_star = np.empty_like(delta_hat)
|
|
366
|
-
n_vec = (
|
|
367
|
-
np.array(list(n_per_batch.values()))
|
|
368
|
-
if isinstance(n_per_batch, dict)
|
|
369
|
-
else n_per_batch
|
|
370
|
-
)
|
|
371
480
|
|
|
372
481
|
for i in range(B):
|
|
373
482
|
n_i = n_vec[i]
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
gamma_star[i] = gamma_post_var * (n_i * g / d + gamma_bar / t2)
|
|
483
|
+
g_hat_i = gamma_hat[i]
|
|
484
|
+
d_hat_i = np.maximum(delta_hat[i], self.eps)
|
|
377
485
|
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
486
|
+
g_new = postmean(g_hat_i, gamma_bar, n_i, d_hat_i, t2)
|
|
487
|
+
sum2 = (n_i - 1) * d_hat_i + n_i * (g_hat_i - g_new) ** 2
|
|
488
|
+
d_new = postvar(sum2, n_i, a_prior, b_prior)
|
|
489
|
+
|
|
490
|
+
converged = False
|
|
491
|
+
n_iter = 0
|
|
492
|
+
gamma_change = float("inf")
|
|
493
|
+
delta_change = float("inf")
|
|
494
|
+
for n_iter in range(1, max_iter + 1): # noqa: B007 # used after loop
|
|
495
|
+
g_prev, d_prev = g_new, d_new
|
|
496
|
+
g_new = postmean(g_hat_i, gamma_bar, n_i, d_prev, t2)
|
|
497
|
+
sum2 = (n_i - 1) * d_hat_i + n_i * (g_hat_i - g_new) ** 2
|
|
498
|
+
d_new = postvar(sum2, n_i, a_prior, b_prior)
|
|
499
|
+
gamma_change = float(
|
|
500
|
+
np.max(np.abs(g_new - g_prev) / (np.abs(g_prev) + self.eps))
|
|
501
|
+
)
|
|
502
|
+
delta_change = float(
|
|
503
|
+
np.max(np.abs(d_new - d_prev) / (np.abs(d_prev) + self.eps))
|
|
504
|
+
)
|
|
505
|
+
if gamma_change < tol and (self.mean_only or delta_change < tol):
|
|
506
|
+
converged = True
|
|
507
|
+
break
|
|
508
|
+
|
|
509
|
+
b_name = batch_names[i] if batch_names else str(i)
|
|
510
|
+
if not converged:
|
|
511
|
+
warnings.warn(
|
|
512
|
+
f"Parametric EB did not converge for batch '{b_name}' "
|
|
513
|
+
f"within {max_iter} iterations "
|
|
514
|
+
f"(final relative change: {max(gamma_change, delta_change):.2e}, "
|
|
515
|
+
f"tol: {tol:.1e}). Results may be unreliable for this batch.",
|
|
516
|
+
stacklevel=2,
|
|
517
|
+
)
|
|
518
|
+
convergence_info.append(
|
|
519
|
+
{
|
|
520
|
+
"batch": b_name,
|
|
521
|
+
"converged": converged,
|
|
522
|
+
"iterations": n_iter,
|
|
523
|
+
"final_gamma_change": gamma_change,
|
|
524
|
+
"final_delta_change": delta_change,
|
|
525
|
+
}
|
|
526
|
+
)
|
|
527
|
+
gamma_star[i] = g_new
|
|
528
|
+
delta_star[i] = 1.0 if self.mean_only else d_new
|
|
529
|
+
self._convergence_info = convergence_info
|
|
381
530
|
return gamma_star, delta_star
|
|
382
531
|
|
|
383
532
|
else:
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
else n_per_batch
|
|
389
|
-
)
|
|
390
|
-
gamma_bar = gamma_hat.mean(axis=0)
|
|
391
|
-
t2 = gamma_hat.var(axis=0, ddof=1)
|
|
392
|
-
|
|
393
|
-
def postmean(
|
|
394
|
-
g_hat: FloatArray,
|
|
395
|
-
g_bar: FloatArray,
|
|
396
|
-
n: float,
|
|
397
|
-
d_star: FloatArray,
|
|
398
|
-
t2_: FloatArray,
|
|
399
|
-
) -> FloatArray:
|
|
400
|
-
return (t2_ * n * g_hat + d_star * g_bar) / (t2_ * n + d_star)
|
|
401
|
-
|
|
402
|
-
def postvar(sum2: FloatArray, n: float, a: FloatArray, b: FloatArray) -> FloatArray:
|
|
403
|
-
return (0.5 * sum2 + b) / (n / 2.0 + a - 1.0)
|
|
404
|
-
|
|
405
|
-
def aprior(delta: FloatArray) -> FloatArray:
|
|
406
|
-
m, s2 = delta.mean(), delta.var()
|
|
407
|
-
s2 = max(s2, self.eps)
|
|
533
|
+
|
|
534
|
+
def aprior(delta: FloatArray) -> float:
|
|
535
|
+
m = float(delta.mean())
|
|
536
|
+
s2 = float(max(delta.var(ddof=1), self.eps))
|
|
408
537
|
return (2 * s2 + m**2) / s2
|
|
409
538
|
|
|
410
|
-
def bprior(delta: FloatArray) ->
|
|
411
|
-
m
|
|
412
|
-
s2 = max(
|
|
539
|
+
def bprior(delta: FloatArray) -> float:
|
|
540
|
+
m = float(delta.mean())
|
|
541
|
+
s2 = float(max(delta.var(ddof=1), self.eps))
|
|
413
542
|
return (m * s2 + m**3) / s2
|
|
414
543
|
|
|
415
544
|
gamma_star = np.empty_like(gamma_hat)
|
|
@@ -422,19 +551,47 @@ class ComBatModel:
|
|
|
422
551
|
a_i = aprior(d_hat_i)
|
|
423
552
|
b_i = bprior(d_hat_i)
|
|
424
553
|
|
|
554
|
+
converged = False
|
|
555
|
+
n_iter = 0
|
|
556
|
+
gamma_change = float("inf")
|
|
557
|
+
delta_change = float("inf")
|
|
425
558
|
g_new, d_new = g_hat_i.copy(), d_hat_i.copy()
|
|
426
|
-
for
|
|
559
|
+
for n_iter in range(1, max_iter + 1): # noqa: B007 # used after loop
|
|
427
560
|
g_prev, d_prev = g_new, d_new
|
|
428
561
|
g_new = postmean(g_hat_i, gamma_bar, n_i, d_prev, t2)
|
|
429
562
|
sum2 = (n_i - 1) * d_hat_i + n_i * (g_hat_i - g_new) ** 2
|
|
430
563
|
d_new = postvar(sum2, n_i, a_i, b_i)
|
|
431
|
-
|
|
432
|
-
self.
|
|
433
|
-
|
|
434
|
-
|
|
564
|
+
gamma_change = float(
|
|
565
|
+
np.max(np.abs(g_new - g_prev) / (np.abs(g_prev) + self.eps))
|
|
566
|
+
)
|
|
567
|
+
delta_change = float(
|
|
568
|
+
np.max(np.abs(d_new - d_prev) / (np.abs(d_prev) + self.eps))
|
|
569
|
+
)
|
|
570
|
+
if gamma_change < tol and (self.mean_only or delta_change < tol):
|
|
571
|
+
converged = True
|
|
435
572
|
break
|
|
573
|
+
|
|
574
|
+
b_name = batch_names[i] if batch_names else str(i)
|
|
575
|
+
if not converged:
|
|
576
|
+
warnings.warn(
|
|
577
|
+
f"Non-parametric EB did not converge for batch '{b_name}' "
|
|
578
|
+
f"within {max_iter} iterations "
|
|
579
|
+
f"(final relative change: {max(gamma_change, delta_change):.2e}, "
|
|
580
|
+
f"tol: {tol:.1e}). Results may be unreliable for this batch.",
|
|
581
|
+
stacklevel=2,
|
|
582
|
+
)
|
|
583
|
+
convergence_info.append(
|
|
584
|
+
{
|
|
585
|
+
"batch": b_name,
|
|
586
|
+
"converged": converged,
|
|
587
|
+
"iterations": n_iter,
|
|
588
|
+
"final_gamma_change": gamma_change,
|
|
589
|
+
"final_delta_change": delta_change,
|
|
590
|
+
}
|
|
591
|
+
)
|
|
436
592
|
gamma_star[i] = g_new
|
|
437
593
|
delta_star[i] = 1.0 if self.mean_only else d_new
|
|
594
|
+
self._convergence_info = convergence_info
|
|
438
595
|
return gamma_star, delta_star
|
|
439
596
|
|
|
440
597
|
def _shrink_gamma(
|
|
@@ -444,10 +601,15 @@ class ComBatModel:
|
|
|
444
601
|
n_per_batch: dict[str, int] | FloatArray,
|
|
445
602
|
*,
|
|
446
603
|
parametric: bool,
|
|
604
|
+
batch_names: list[str] | None = None,
|
|
447
605
|
) -> FloatArray:
|
|
448
606
|
"""Convenience wrapper that returns only gamma* (for *mean-only* mode)."""
|
|
449
607
|
gamma, _ = self._shrink_gamma_delta(
|
|
450
|
-
gamma_hat,
|
|
608
|
+
gamma_hat,
|
|
609
|
+
delta_hat,
|
|
610
|
+
n_per_batch,
|
|
611
|
+
parametric=parametric,
|
|
612
|
+
batch_names=batch_names,
|
|
451
613
|
)
|
|
452
614
|
return gamma
|
|
453
615
|
|
|
@@ -459,7 +621,29 @@ class ComBatModel:
|
|
|
459
621
|
discrete_covariates: ArrayLike | None = None,
|
|
460
622
|
continuous_covariates: ArrayLike | None = None,
|
|
461
623
|
) -> pd.DataFrame:
|
|
462
|
-
"""Transform the data using fitted ComBat parameters.
|
|
624
|
+
"""Transform the data using fitted ComBat parameters.
|
|
625
|
+
|
|
626
|
+
Parameters
|
|
627
|
+
----------
|
|
628
|
+
X : array-like of shape (n_samples, n_features)
|
|
629
|
+
Input data to transform.
|
|
630
|
+
batch : array-like of shape (n_samples,)
|
|
631
|
+
Batch labels for each sample.
|
|
632
|
+
discrete_covariates : array-like or None, default=None
|
|
633
|
+
Categorical covariates (Fortin/Chen only).
|
|
634
|
+
continuous_covariates : array-like or None, default=None
|
|
635
|
+
Continuous covariates (Fortin/Chen only).
|
|
636
|
+
|
|
637
|
+
Returns
|
|
638
|
+
-------
|
|
639
|
+
X_adjusted : pd.DataFrame
|
|
640
|
+
Batch-corrected data.
|
|
641
|
+
|
|
642
|
+
Raises
|
|
643
|
+
------
|
|
644
|
+
ValueError
|
|
645
|
+
If the model is not fitted or if unseen batch levels are present.
|
|
646
|
+
"""
|
|
463
647
|
if not hasattr(self, "_gamma_star"):
|
|
464
648
|
raise ValueError(
|
|
465
649
|
"This ComBatModel instance is not fitted yet. Call 'fit' before 'transform'."
|
|
@@ -482,7 +666,9 @@ class ComBatModel:
|
|
|
482
666
|
elif method == "chen":
|
|
483
667
|
return self._transform_chen(X, batch, disc, cont)
|
|
484
668
|
else:
|
|
485
|
-
raise ValueError(
|
|
669
|
+
raise ValueError(
|
|
670
|
+
f"Unknown method={method!r}. Expected one of 'johnson', 'fortin', or 'chen'."
|
|
671
|
+
)
|
|
486
672
|
|
|
487
673
|
def _transform_johnson(self, X: pd.DataFrame, batch: pd.Series) -> pd.DataFrame:
|
|
488
674
|
"""Johnson transform implementation."""
|
|
@@ -514,21 +700,32 @@ class ComBatModel:
|
|
|
514
700
|
cont: pd.DataFrame | None,
|
|
515
701
|
) -> pd.DataFrame:
|
|
516
702
|
"""Fortin transform implementation."""
|
|
517
|
-
batch_dummies = pd.get_dummies(batch, drop_first=False).astype(float)
|
|
703
|
+
batch_dummies = pd.get_dummies(batch, drop_first=False).astype(float)
|
|
704
|
+
batch_dummies = batch_dummies.reindex(columns=self._batch_levels, fill_value=0.0)
|
|
518
705
|
if self.reference_batch is not None:
|
|
519
706
|
batch_dummies.loc[:, self.reference_batch] = 1.0
|
|
520
707
|
|
|
521
|
-
|
|
708
|
+
nonbatch_parts: list[pd.DataFrame] = []
|
|
522
709
|
if disc is not None:
|
|
523
|
-
|
|
710
|
+
nonbatch_parts.append(
|
|
711
|
+
pd.get_dummies(disc.astype("category"), drop_first=True).astype(float)
|
|
712
|
+
)
|
|
524
713
|
if cont is not None:
|
|
525
|
-
|
|
714
|
+
nonbatch_parts.append(cont.astype(float))
|
|
715
|
+
|
|
716
|
+
if nonbatch_parts:
|
|
717
|
+
nonbatch_df = pd.concat(nonbatch_parts, axis=1)
|
|
718
|
+
else:
|
|
719
|
+
nonbatch_df = pd.DataFrame(index=batch_dummies.index)
|
|
720
|
+
nonbatch_df = nonbatch_df.reindex(columns=self._nonbatch_columns, fill_value=0.0)
|
|
526
721
|
|
|
527
|
-
design = pd.concat(
|
|
722
|
+
design = pd.concat([batch_dummies, nonbatch_df], axis=1).values
|
|
528
723
|
|
|
529
724
|
X_np = X.values
|
|
530
|
-
|
|
531
|
-
|
|
725
|
+
grand_vals = np.asarray(self._grand_mean.values, dtype=np.float64)
|
|
726
|
+
pooled_vals = np.asarray(self._pooled_var.values, dtype=np.float64)
|
|
727
|
+
stand_mu = grand_vals + design[:, self._n_batch :] @ self._beta_hat_nonbatch
|
|
728
|
+
Xs = (X_np - stand_mu) / np.sqrt(pooled_vals)
|
|
532
729
|
|
|
533
730
|
for i, lvl in enumerate(self._batch_levels):
|
|
534
731
|
idx = batch == lvl
|
|
@@ -545,7 +742,7 @@ class ComBatModel:
|
|
|
545
742
|
else:
|
|
546
743
|
Xs[idx] = (Xs[idx] - g) / np.sqrt(d)
|
|
547
744
|
|
|
548
|
-
X_adj = Xs * np.sqrt(
|
|
745
|
+
X_adj = Xs * np.sqrt(pooled_vals) + stand_mu
|
|
549
746
|
return pd.DataFrame(X_adj, index=X.index, columns=X.columns, dtype=float)
|
|
550
747
|
|
|
551
748
|
def _transform_chen(
|