combatlearn 2.0.0__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {combatlearn-2.0.0 → combatlearn-2.1.0}/PKG-INFO +13 -6
- {combatlearn-2.0.0 → combatlearn-2.1.0}/README.md +12 -6
- {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn/__init__.py +1 -1
- combatlearn-2.1.0/combatlearn/_mixed.py +165 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn/core.py +243 -10
- {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn/sklearn_api.py +42 -6
- {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn.egg-info/PKG-INFO +13 -6
- {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn.egg-info/SOURCES.txt +2 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn.egg-info/requires.txt +1 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/requirements-dev.txt +1 -0
- combatlearn-2.1.0/tests/test_longitudinal.py +204 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/LICENSE +0 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn/_utils.py +0 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn/inspection.py +0 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn/metrics.py +0 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn/py.typed +0 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn/visualization.py +0 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn.egg-info/dependency_links.txt +0 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn.egg-info/top_level.txt +0 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/pyproject.toml +0 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/requirements-docs.txt +0 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/requirements.txt +0 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/setup.cfg +0 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/tests/test_combat.py +0 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/tests/test_inspection.py +0 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/tests/test_metrics.py +0 -0
- {combatlearn-2.0.0 → combatlearn-2.1.0}/tests/test_visualization.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: combatlearn
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: Batch-effect harmonization for machine learning frameworks.
|
|
5
5
|
Author-email: Ettore Rocchi <ettoreroc@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -27,6 +27,7 @@ Requires-Dist: ruff>=0.8; extra == "dev"
|
|
|
27
27
|
Requires-Dist: pre-commit>=3.0; extra == "dev"
|
|
28
28
|
Requires-Dist: mypy>=1.0; extra == "dev"
|
|
29
29
|
Requires-Dist: pandas-stubs; extra == "dev"
|
|
30
|
+
Requires-Dist: statsmodels>=0.14; extra == "dev"
|
|
30
31
|
Provides-Extra: docs
|
|
31
32
|
Requires-Dist: sphinx>=7.0.0; extra == "docs"
|
|
32
33
|
Requires-Dist: pydata-sphinx-theme>=0.15.0; extra == "docs"
|
|
@@ -49,10 +50,11 @@ Dynamic: license-file
|
|
|
49
50
|
|
|
50
51
|
**combatlearn** makes the popular _ComBat_ (and _CovBat_) batch-effect correction algorithm available for use into machine learning frameworks. It lets you harmonise high-dimensional data inside a scikit-learn `Pipeline`, so that cross-validation and grid-search automatically take batch structure into account, **without data leakage**.
|
|
51
52
|
|
|
52
|
-
**
|
|
53
|
+
**Four methods**:
|
|
53
54
|
- `method="johnson"` - classic ComBat (Johnson _et al._, 2007)
|
|
54
55
|
- `method="fortin"` - neuroComBat (Fortin _et al._, 2018)
|
|
55
56
|
- `method="chen"` - CovBat (Chen _et al._, 2022)
|
|
57
|
+
- `method="longitudinal"` - Longitudinal ComBat for repeated measures (Beer _et al._, 2020)
|
|
56
58
|
|
|
57
59
|
## Installation
|
|
58
60
|
|
|
@@ -75,6 +77,7 @@ import pandas as pd
|
|
|
75
77
|
from sklearn.pipeline import Pipeline
|
|
76
78
|
from sklearn.preprocessing import StandardScaler
|
|
77
79
|
from sklearn.linear_model import LogisticRegression
|
|
80
|
+
from sklearn.model_selection import GridSearchCV
|
|
78
81
|
from combatlearn import ComBat
|
|
79
82
|
|
|
80
83
|
df = pd.read_csv("data.csv", index_col=0)
|
|
@@ -125,18 +128,20 @@ The following section provides a detailed explanation of all parameters availabl
|
|
|
125
128
|
| Parameter | Type | Default | Description |
|
|
126
129
|
| --- | --- | --- | --- |
|
|
127
130
|
| `batch` | array-like or pd.Series | **required** | Vector indicating batch assignment for each sample. This is used to estimate and remove batch effects. |
|
|
128
|
-
| `discrete_covariates` | array-like, pd.Series, or pd.DataFrame | `None` | Optional categorical covariates (e.g., sex, site). Only used in `"fortin"` and `"
|
|
129
|
-
| `continuous_covariates` | array-like, pd.Series or pd.DataFrame | `None` | Optional continuous covariates (e.g., age). Only used in `"fortin"` and `"
|
|
131
|
+
| `discrete_covariates` | array-like, pd.Series, or pd.DataFrame | `None` | Optional categorical covariates (e.g., sex, site). Only used in `"fortin"`, `"chen"`, and `"longitudinal"` methods. |
|
|
132
|
+
| `continuous_covariates` | array-like, pd.Series or pd.DataFrame | `None` | Optional continuous covariates (e.g., age). Only used in `"fortin"`, `"chen"`, and `"longitudinal"` methods. |
|
|
133
|
+
| `subject_id` | array-like or pd.Series | `None` | Subject/individual labels for the random intercept. **Required** for `method="longitudinal"`, ignored otherwise. |
|
|
134
|
+
| `time_covariate` | array-like or pd.Series | `None` | Optional continuous time variable for repeated measures. Only used in `"longitudinal"`. |
|
|
130
135
|
|
|
131
136
|
### Algorithm Options
|
|
132
137
|
|
|
133
138
|
| Parameter | Type | Default | Description |
|
|
134
139
|
| --- | --- | --- | --- |
|
|
135
|
-
| `method` | str | `"johnson"` | ComBat method to use: <ul><li>`"johnson"` - Classical ComBat (_Johnson et al. 2007_)</li><li>`"fortin"` - ComBat with covariates (_Fortin et al. 2018_)</li><li>`"chen"` - CovBat, PCA-based correction (_Chen et al. 2022_)</li></ul> |
|
|
140
|
+
| `method` | str | `"johnson"` | ComBat method to use: <ul><li>`"johnson"` - Classical ComBat (_Johnson et al. 2007_)</li><li>`"fortin"` - ComBat with covariates (_Fortin et al. 2018_)</li><li>`"chen"` - CovBat, PCA-based correction (_Chen et al. 2022_)</li><li>`"longitudinal"` - Longitudinal ComBat with a per-subject random intercept (_Beer et al. 2020_)</li></ul> |
|
|
136
141
|
| `parametric` | bool | `True` | Whether to use the **parametric empirical Bayes** formulation. If `False`, a non-parametric iterative scheme is used. |
|
|
137
142
|
| `mean_only` | bool | `False` | If `True`, only the **mean** is corrected, while variances are left unchanged. Useful for preserving variance structure in the data. |
|
|
138
143
|
| `reference_batch` | str or `None` | `None` | If specified, acts as a reference batch - other batches will be corrected to match this one. |
|
|
139
|
-
| `covbat_cov_thresh` | float, int | `0.9` | For `"chen"` method only: Cumulative variance threshold
|
|
144
|
+
| `covbat_cov_thresh` | float, int | `0.9` | For `"chen"` method only: Cumulative variance threshold in `(0, 1]` to retain PCs in PCA space (e.g., 0.9 = retain 90% explained variance). If an integer is provided, it represents the number of principal components to use. |
|
|
140
145
|
| `eps` | float | `1e-8` | Small jitter value added to variances to prevent divide-by-zero errors during standardization. |
|
|
141
146
|
|
|
142
147
|
|
|
@@ -190,3 +195,5 @@ Please consider citing the original papers:
|
|
|
190
195
|
- [**neuroCombat**](https://github.com/Jfortin1/neuroCombat) - Fortin JP et al. _Neuroimage_. 2018. doi: [10.1016/j.neuroimage.2017.11.024](https://doi.org/10.1016/j.neuroimage.2017.11.024)
|
|
191
196
|
|
|
192
197
|
- [**CovBat**](https://github.com/andy1764/CovBat_Harmonization) - Chen AA et al. _Hum Brain Mapp_. 2022. doi: [10.1002/hbm.25688](https://doi.org/10.1002/hbm.25688)
|
|
198
|
+
|
|
199
|
+
- [**Longitudinal ComBat**](https://github.com/jcbeer/longCombat) - Beer JC et al. _Neuroimage_. 2020. doi: [10.1016/j.neuroimage.2020.117129](https://doi.org/10.1016/j.neuroimage.2020.117129)
|
|
@@ -12,10 +12,11 @@
|
|
|
12
12
|
|
|
13
13
|
**combatlearn** makes the popular _ComBat_ (and _CovBat_) batch-effect correction algorithm available for use into machine learning frameworks. It lets you harmonise high-dimensional data inside a scikit-learn `Pipeline`, so that cross-validation and grid-search automatically take batch structure into account, **without data leakage**.
|
|
14
14
|
|
|
15
|
-
**
|
|
15
|
+
**Four methods**:
|
|
16
16
|
- `method="johnson"` - classic ComBat (Johnson _et al._, 2007)
|
|
17
17
|
- `method="fortin"` - neuroComBat (Fortin _et al._, 2018)
|
|
18
18
|
- `method="chen"` - CovBat (Chen _et al._, 2022)
|
|
19
|
+
- `method="longitudinal"` - Longitudinal ComBat for repeated measures (Beer _et al._, 2020)
|
|
19
20
|
|
|
20
21
|
## Installation
|
|
21
22
|
|
|
@@ -38,6 +39,7 @@ import pandas as pd
|
|
|
38
39
|
from sklearn.pipeline import Pipeline
|
|
39
40
|
from sklearn.preprocessing import StandardScaler
|
|
40
41
|
from sklearn.linear_model import LogisticRegression
|
|
42
|
+
from sklearn.model_selection import GridSearchCV
|
|
41
43
|
from combatlearn import ComBat
|
|
42
44
|
|
|
43
45
|
df = pd.read_csv("data.csv", index_col=0)
|
|
@@ -88,18 +90,20 @@ The following section provides a detailed explanation of all parameters availabl
|
|
|
88
90
|
| Parameter | Type | Default | Description |
|
|
89
91
|
| --- | --- | --- | --- |
|
|
90
92
|
| `batch` | array-like or pd.Series | **required** | Vector indicating batch assignment for each sample. This is used to estimate and remove batch effects. |
|
|
91
|
-
| `discrete_covariates` | array-like, pd.Series, or pd.DataFrame | `None` | Optional categorical covariates (e.g., sex, site). Only used in `"fortin"` and `"
|
|
92
|
-
| `continuous_covariates` | array-like, pd.Series or pd.DataFrame | `None` | Optional continuous covariates (e.g., age). Only used in `"fortin"` and `"
|
|
93
|
+
| `discrete_covariates` | array-like, pd.Series, or pd.DataFrame | `None` | Optional categorical covariates (e.g., sex, site). Only used in `"fortin"`, `"chen"`, and `"longitudinal"` methods. |
|
|
94
|
+
| `continuous_covariates` | array-like, pd.Series or pd.DataFrame | `None` | Optional continuous covariates (e.g., age). Only used in `"fortin"`, `"chen"`, and `"longitudinal"` methods. |
|
|
95
|
+
| `subject_id` | array-like or pd.Series | `None` | Subject/individual labels for the random intercept. **Required** for `method="longitudinal"`, ignored otherwise. |
|
|
96
|
+
| `time_covariate` | array-like or pd.Series | `None` | Optional continuous time variable for repeated measures. Only used in `"longitudinal"`. |
|
|
93
97
|
|
|
94
98
|
### Algorithm Options
|
|
95
99
|
|
|
96
100
|
| Parameter | Type | Default | Description |
|
|
97
101
|
| --- | --- | --- | --- |
|
|
98
|
-
| `method` | str | `"johnson"` | ComBat method to use: <ul><li>`"johnson"` - Classical ComBat (_Johnson et al. 2007_)</li><li>`"fortin"` - ComBat with covariates (_Fortin et al. 2018_)</li><li>`"chen"` - CovBat, PCA-based correction (_Chen et al. 2022_)</li></ul> |
|
|
102
|
+
| `method` | str | `"johnson"` | ComBat method to use: <ul><li>`"johnson"` - Classical ComBat (_Johnson et al. 2007_)</li><li>`"fortin"` - ComBat with covariates (_Fortin et al. 2018_)</li><li>`"chen"` - CovBat, PCA-based correction (_Chen et al. 2022_)</li><li>`"longitudinal"` - Longitudinal ComBat with a per-subject random intercept (_Beer et al. 2020_)</li></ul> |
|
|
99
103
|
| `parametric` | bool | `True` | Whether to use the **parametric empirical Bayes** formulation. If `False`, a non-parametric iterative scheme is used. |
|
|
100
104
|
| `mean_only` | bool | `False` | If `True`, only the **mean** is corrected, while variances are left unchanged. Useful for preserving variance structure in the data. |
|
|
101
105
|
| `reference_batch` | str or `None` | `None` | If specified, acts as a reference batch - other batches will be corrected to match this one. |
|
|
102
|
-
| `covbat_cov_thresh` | float, int | `0.9` | For `"chen"` method only: Cumulative variance threshold
|
|
106
|
+
| `covbat_cov_thresh` | float, int | `0.9` | For `"chen"` method only: Cumulative variance threshold in `(0, 1]` to retain PCs in PCA space (e.g., 0.9 = retain 90% explained variance). If an integer is provided, it represents the number of principal components to use. |
|
|
103
107
|
| `eps` | float | `1e-8` | Small jitter value added to variances to prevent divide-by-zero errors during standardization. |
|
|
104
108
|
|
|
105
109
|
|
|
@@ -152,4 +156,6 @@ Please consider citing the original papers:
|
|
|
152
156
|
|
|
153
157
|
- [**neuroCombat**](https://github.com/Jfortin1/neuroCombat) - Fortin JP et al. _Neuroimage_. 2018. doi: [10.1016/j.neuroimage.2017.11.024](https://doi.org/10.1016/j.neuroimage.2017.11.024)
|
|
154
158
|
|
|
155
|
-
- [**CovBat**](https://github.com/andy1764/CovBat_Harmonization) - Chen AA et al. _Hum Brain Mapp_. 2022. doi: [10.1002/hbm.25688](https://doi.org/10.1002/hbm.25688)
|
|
159
|
+
- [**CovBat**](https://github.com/andy1764/CovBat_Harmonization) - Chen AA et al. _Hum Brain Mapp_. 2022. doi: [10.1002/hbm.25688](https://doi.org/10.1002/hbm.25688)
|
|
160
|
+
|
|
161
|
+
- [**Longitudinal ComBat**](https://github.com/jcbeer/longCombat) - Beer JC et al. _Neuroimage_. 2020. doi: [10.1016/j.neuroimage.2020.117129](https://doi.org/10.1016/j.neuroimage.2020.117129)
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""Random-intercept linear mixed model fitting for Longitudinal ComBat.
|
|
2
|
+
|
|
3
|
+
Fits, for every feature (column of ``Y``) simultaneously, the model
|
|
4
|
+
|
|
5
|
+
y = X @ beta + Z @ u + eps, u ~ N(0, tau^2 I), eps ~ N(0, sigma^2 I)
|
|
6
|
+
|
|
7
|
+
where ``Z`` is the subject (random-intercept) indicator matrix. Estimation is
|
|
8
|
+
by REML, profiling out ``beta`` and ``sigma^2`` so that only the scalar
|
|
9
|
+
variance ratio ``lambda = tau^2 / sigma^2`` is optimized per feature.
|
|
10
|
+
|
|
11
|
+
The single random-intercept grouping makes the marginal covariance
|
|
12
|
+
``V = sigma^2 (I + lambda Z Z^T)`` block diagonal (one block per subject), so
|
|
13
|
+
``V^{-1}`` is available in closed form via Sherman-Morrison and every operation
|
|
14
|
+
is O(n) per feature. The variance ratio is found by a vectorized grid search
|
|
15
|
+
over ``lambda`` followed by a parabolic refinement, after which the fixed
|
|
16
|
+
effects, residual variance, and random-intercept BLUPs are computed in a single
|
|
17
|
+
per-feature pass.
|
|
18
|
+
|
|
19
|
+
This keeps Longitudinal ComBat dependency-free (numpy only); correctness is
|
|
20
|
+
cross-checked against ``statsmodels.MixedLM`` in the test suite.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import numpy as np
|
|
26
|
+
import numpy.typing as npt
|
|
27
|
+
|
|
28
|
+
FloatArray = npt.NDArray[np.float64]
|
|
29
|
+
IntArray = npt.NDArray[np.intp]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _group_sums(M: FloatArray, group_idx: IntArray, n_groups: int) -> FloatArray:
|
|
33
|
+
"""Sum the rows of ``M`` (n, c) within each group, returning (n_groups, c)."""
|
|
34
|
+
out = np.zeros((n_groups, M.shape[1]), dtype=np.float64)
|
|
35
|
+
np.add.at(out, group_idx, M)
|
|
36
|
+
return out
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _w_apply(M: FloatArray, factor: FloatArray, group_idx: IntArray, n_groups: int) -> FloatArray:
|
|
40
|
+
"""Apply ``W = (I + lambda Z Z^T)^{-1}`` to the columns of ``M``.
|
|
41
|
+
|
|
42
|
+
``factor[k] = lambda / (1 + lambda * n_k)`` is the Sherman-Morrison
|
|
43
|
+
per-group correction (``n_k`` = number of observations in group ``k``).
|
|
44
|
+
"""
|
|
45
|
+
gs = _group_sums(M, group_idx, n_groups)
|
|
46
|
+
return M - factor[group_idx][:, None] * gs[group_idx]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _reml_objective(
|
|
50
|
+
lam: float,
|
|
51
|
+
X: FloatArray,
|
|
52
|
+
Y: FloatArray,
|
|
53
|
+
group_idx: IntArray,
|
|
54
|
+
n_groups: int,
|
|
55
|
+
n_k: FloatArray,
|
|
56
|
+
n: int,
|
|
57
|
+
p: int,
|
|
58
|
+
ridge: float,
|
|
59
|
+
) -> FloatArray:
|
|
60
|
+
"""Profiled REML objective (to minimize) evaluated for all features at ``lam``."""
|
|
61
|
+
factor = lam / (1.0 + lam * n_k)
|
|
62
|
+
WX = _w_apply(X, factor, group_idx, n_groups)
|
|
63
|
+
A = X.T @ WX
|
|
64
|
+
A.flat[:: p + 1] += ridge
|
|
65
|
+
beta = np.linalg.solve(A, X.T @ _w_apply(Y, factor, group_idx, n_groups))
|
|
66
|
+
R = Y - X @ beta
|
|
67
|
+
WR = _w_apply(R, factor, group_idx, n_groups)
|
|
68
|
+
rss = np.maximum(np.einsum("ng,ng->g", R, WR), 1e-12)
|
|
69
|
+
logdet_v = float(np.sum(np.log1p(lam * n_k)))
|
|
70
|
+
_, logdet_a = np.linalg.slogdet(A)
|
|
71
|
+
result = logdet_v + (n - p) * np.log(rss) + float(logdet_a)
|
|
72
|
+
return np.asarray(result, dtype=np.float64)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def fit_random_intercept(
|
|
76
|
+
X: FloatArray,
|
|
77
|
+
Y: FloatArray,
|
|
78
|
+
group_idx: IntArray,
|
|
79
|
+
n_groups: int,
|
|
80
|
+
n_k: FloatArray,
|
|
81
|
+
*,
|
|
82
|
+
eps: float = 1e-8,
|
|
83
|
+
n_grid: int = 80,
|
|
84
|
+
) -> tuple[FloatArray, FloatArray, FloatArray, FloatArray]:
|
|
85
|
+
"""Fit a random-intercept LME per feature by REML.
|
|
86
|
+
|
|
87
|
+
Parameters
|
|
88
|
+
----------
|
|
89
|
+
X : ndarray of shape (n, p)
|
|
90
|
+
Fixed-effects design matrix (shared across features).
|
|
91
|
+
Y : ndarray of shape (n, n_features)
|
|
92
|
+
Response matrix; one column per feature.
|
|
93
|
+
group_idx : ndarray of shape (n,)
|
|
94
|
+
Integer group (subject) code in ``[0, n_groups)`` for each observation.
|
|
95
|
+
n_groups : int
|
|
96
|
+
Number of distinct subjects.
|
|
97
|
+
n_k : ndarray of shape (n_groups,)
|
|
98
|
+
Number of observations per subject.
|
|
99
|
+
eps : float, default=1e-8
|
|
100
|
+
Numerical jitter; also scales the diagonal ridge on ``X^T W X``.
|
|
101
|
+
n_grid : int, default=80
|
|
102
|
+
Number of log-spaced ``lambda`` grid points for the variance-ratio search.
|
|
103
|
+
|
|
104
|
+
Returns
|
|
105
|
+
-------
|
|
106
|
+
beta : ndarray of shape (p, n_features)
|
|
107
|
+
Estimated fixed-effects coefficients.
|
|
108
|
+
blup : ndarray of shape (n_groups, n_features)
|
|
109
|
+
Best linear unbiased predictors of the subject random intercepts.
|
|
110
|
+
sigma2 : ndarray of shape (n_features,)
|
|
111
|
+
Estimated residual variance per feature.
|
|
112
|
+
lam_opt : ndarray of shape (n_features,)
|
|
113
|
+
Estimated variance ratio ``tau^2 / sigma^2`` per feature.
|
|
114
|
+
"""
|
|
115
|
+
X = np.asarray(X, dtype=np.float64)
|
|
116
|
+
Y = np.asarray(Y, dtype=np.float64)
|
|
117
|
+
group_idx = np.asarray(group_idx, dtype=np.intp)
|
|
118
|
+
n_k = np.asarray(n_k, dtype=np.float64)
|
|
119
|
+
n, p = X.shape
|
|
120
|
+
n_features = Y.shape[1]
|
|
121
|
+
ridge = eps * float(np.trace(X.T @ X)) / max(p, 1)
|
|
122
|
+
|
|
123
|
+
# Vectorized grid search for the variance ratio (per feature).
|
|
124
|
+
log_grid = np.linspace(-4.0, 4.0, n_grid)
|
|
125
|
+
lam_grid = np.concatenate([[0.0], 10.0**log_grid])
|
|
126
|
+
fvals = np.empty((lam_grid.size, n_features), dtype=np.float64)
|
|
127
|
+
for i, lam in enumerate(lam_grid):
|
|
128
|
+
fvals[i] = _reml_objective(lam, X, Y, group_idx, n_groups, n_k, n, p, ridge)
|
|
129
|
+
best = np.argmin(fvals, axis=0)
|
|
130
|
+
lam_opt = lam_grid[best].astype(np.float64)
|
|
131
|
+
|
|
132
|
+
# Vectorized parabolic refinement (log10 space) for interior positive optima.
|
|
133
|
+
interior = (best >= 2) & (best <= n_grid - 1)
|
|
134
|
+
if np.any(interior):
|
|
135
|
+
bi = best[interior]
|
|
136
|
+
f_lo = fvals[bi - 1, interior]
|
|
137
|
+
f0 = fvals[bi, interior]
|
|
138
|
+
f_hi = fvals[bi + 1, interior]
|
|
139
|
+
denom = f_lo - 2.0 * f0 + f_hi
|
|
140
|
+
delta = np.where(np.abs(denom) > 1e-12, 0.5 * (f_lo - f_hi) / denom, 0.0)
|
|
141
|
+
delta = np.clip(delta, -1.0, 1.0)
|
|
142
|
+
step = log_grid[1] - log_grid[0]
|
|
143
|
+
lam_opt[interior] = 10.0 ** (log_grid[bi - 1] + delta * step)
|
|
144
|
+
|
|
145
|
+
# Final per-feature fixed effects, residual variance, and random-intercept BLUPs.
|
|
146
|
+
beta = np.empty((p, n_features), dtype=np.float64)
|
|
147
|
+
sigma2 = np.empty(n_features, dtype=np.float64)
|
|
148
|
+
blup = np.zeros((n_groups, n_features), dtype=np.float64)
|
|
149
|
+
denom_sigma = max(n - p, 1)
|
|
150
|
+
for g in range(n_features):
|
|
151
|
+
lam = float(lam_opt[g])
|
|
152
|
+
factor = lam / (1.0 + lam * n_k)
|
|
153
|
+
A = X.T @ _w_apply(X, factor, group_idx, n_groups)
|
|
154
|
+
A.flat[:: p + 1] += ridge
|
|
155
|
+
y = Y[:, g : g + 1]
|
|
156
|
+
b = np.linalg.solve(A, X.T @ _w_apply(y, factor, group_idx, n_groups))[:, 0]
|
|
157
|
+
beta[:, g] = b
|
|
158
|
+
r = Y[:, g] - X @ b
|
|
159
|
+
Wr = _w_apply(r[:, None], factor, group_idx, n_groups)[:, 0]
|
|
160
|
+
sigma2[g] = float(r @ Wr) / denom_sigma
|
|
161
|
+
s_k = np.zeros(n_groups, dtype=np.float64)
|
|
162
|
+
np.add.at(s_k, group_idx, r)
|
|
163
|
+
blup[:, g] = lam * s_k / (1.0 + lam * n_k)
|
|
164
|
+
|
|
165
|
+
return beta, blup, sigma2, lam_opt
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
"""ComBat algorithm core.
|
|
2
2
|
|
|
3
|
-
`ComBatModel` implements
|
|
3
|
+
`ComBatModel` implements four variants of the ComBat algorithm:
|
|
4
4
|
* Johnson et al. (2007) vanilla ComBat (method="johnson")
|
|
5
5
|
* Fortin et al. (2018) extension with covariates (method="fortin")
|
|
6
6
|
* Chen et al. (2022) CovBat (method="chen")
|
|
7
|
+
* Beer et al. (2020) Longitudinal ComBat (method="longitudinal")
|
|
7
8
|
"""
|
|
8
9
|
|
|
9
10
|
from __future__ import annotations
|
|
@@ -17,6 +18,8 @@ import numpy.typing as npt
|
|
|
17
18
|
import pandas as pd
|
|
18
19
|
from sklearn.decomposition import PCA
|
|
19
20
|
|
|
21
|
+
from ._mixed import fit_random_intercept
|
|
22
|
+
|
|
20
23
|
ArrayLike: TypeAlias = pd.DataFrame | pd.Series | npt.NDArray[Any]
|
|
21
24
|
FloatArray: TypeAlias = npt.NDArray[np.float64]
|
|
22
25
|
|
|
@@ -26,10 +29,12 @@ class ComBatModel:
|
|
|
26
29
|
|
|
27
30
|
Parameters
|
|
28
31
|
----------
|
|
29
|
-
method : {'johnson', 'fortin', 'chen'}, default='johnson'
|
|
32
|
+
method : {'johnson', 'fortin', 'chen', 'longitudinal'}, default='johnson'
|
|
30
33
|
* 'johnson' - classic ComBat.
|
|
31
34
|
* 'fortin' - covariate-aware ComBat.
|
|
32
35
|
* 'chen' - CovBat, PCA-based ComBat.
|
|
36
|
+
* 'longitudinal' - Longitudinal ComBat; requires a per-sample
|
|
37
|
+
``subject_id`` for the random intercept.
|
|
33
38
|
parametric : bool, default=True
|
|
34
39
|
Use the parametric empirical Bayes variant.
|
|
35
40
|
mean_only : bool, default=False
|
|
@@ -47,7 +52,7 @@ class ComBatModel:
|
|
|
47
52
|
def __init__(
|
|
48
53
|
self,
|
|
49
54
|
*,
|
|
50
|
-
method: Literal["johnson", "fortin", "chen"] = "johnson",
|
|
55
|
+
method: Literal["johnson", "fortin", "chen", "longitudinal"] = "johnson",
|
|
51
56
|
parametric: bool = True,
|
|
52
57
|
mean_only: bool = False,
|
|
53
58
|
reference_batch: str | None = None,
|
|
@@ -77,6 +82,8 @@ class ComBatModel:
|
|
|
77
82
|
self._batch_levels_pc: pd.Index
|
|
78
83
|
self._pc_gamma_star: FloatArray
|
|
79
84
|
self._pc_delta_star: FloatArray
|
|
85
|
+
self._re_subject_levels: pd.Index
|
|
86
|
+
self._re_blup: FloatArray
|
|
80
87
|
|
|
81
88
|
# Validate covbat_cov_thresh
|
|
82
89
|
if isinstance(self.covbat_cov_thresh, float):
|
|
@@ -138,6 +145,8 @@ class ComBatModel:
|
|
|
138
145
|
batch: ArrayLike,
|
|
139
146
|
discrete_covariates: ArrayLike | None = None,
|
|
140
147
|
continuous_covariates: ArrayLike | None = None,
|
|
148
|
+
subject_id: ArrayLike | None = None,
|
|
149
|
+
time_covariate: ArrayLike | None = None,
|
|
141
150
|
) -> ComBatModel:
|
|
142
151
|
"""Fit the ComBat model.
|
|
143
152
|
|
|
@@ -150,9 +159,15 @@ class ComBatModel:
|
|
|
150
159
|
batch : array-like of shape (n_samples,)
|
|
151
160
|
Batch labels for each sample.
|
|
152
161
|
discrete_covariates : array-like or None, default=None
|
|
153
|
-
Categorical covariates to protect (Fortin/Chen only).
|
|
162
|
+
Categorical covariates to protect (Fortin/Chen/Longitudinal only).
|
|
154
163
|
continuous_covariates : array-like or None, default=None
|
|
155
|
-
Continuous covariates to protect (Fortin/Chen only).
|
|
164
|
+
Continuous covariates to protect (Fortin/Chen/Longitudinal only).
|
|
165
|
+
subject_id : array-like or None, default=None
|
|
166
|
+
Subject/individual labels for the random intercept. Required for
|
|
167
|
+
``method='longitudinal'``, ignored otherwise.
|
|
168
|
+
time_covariate : array-like or None, default=None
|
|
169
|
+
Continuous time variable for repeated measures (Longitudinal only).
|
|
170
|
+
Added to the fixed-effects design alongside ``continuous_covariates``.
|
|
156
171
|
|
|
157
172
|
Returns
|
|
158
173
|
-------
|
|
@@ -160,10 +175,10 @@ class ComBatModel:
|
|
|
160
175
|
Fitted model.
|
|
161
176
|
"""
|
|
162
177
|
method = self.method.lower()
|
|
163
|
-
if method not in {"johnson", "fortin", "chen"}:
|
|
178
|
+
if method not in {"johnson", "fortin", "chen", "longitudinal"}:
|
|
164
179
|
raise ValueError(
|
|
165
180
|
f"method={self.method!r} is not recognized. "
|
|
166
|
-
f"Expected one of 'johnson', 'fortin', or '
|
|
181
|
+
f"Expected one of 'johnson', 'fortin', 'chen', or 'longitudinal'."
|
|
167
182
|
)
|
|
168
183
|
if not isinstance(X, pd.DataFrame):
|
|
169
184
|
X = pd.DataFrame(X)
|
|
@@ -180,6 +195,13 @@ class ComBatModel:
|
|
|
180
195
|
f"Check for typos or trailing whitespace in batch labels."
|
|
181
196
|
)
|
|
182
197
|
|
|
198
|
+
if method != "longitudinal" and (subject_id is not None or time_covariate is not None):
|
|
199
|
+
warnings.warn(
|
|
200
|
+
"`subject_id` and `time_covariate` are only used when "
|
|
201
|
+
"method='longitudinal'; they are ignored here.",
|
|
202
|
+
stacklevel=2,
|
|
203
|
+
)
|
|
204
|
+
|
|
183
205
|
if method == "johnson":
|
|
184
206
|
if disc is not None or cont is not None:
|
|
185
207
|
warnings.warn("Covariates are ignored when using method='johnson'.", stacklevel=2)
|
|
@@ -188,6 +210,8 @@ class ComBatModel:
|
|
|
188
210
|
self._fit_fortin(X, batch, disc, cont)
|
|
189
211
|
elif method == "chen":
|
|
190
212
|
self._fit_chen(X, batch, disc, cont)
|
|
213
|
+
elif method == "longitudinal":
|
|
214
|
+
self._fit_longitudinal(X, batch, disc, cont, subject_id, time_covariate)
|
|
191
215
|
return self
|
|
192
216
|
|
|
193
217
|
def _fit_johnson(self, X: pd.DataFrame, batch: pd.Series) -> None:
|
|
@@ -431,6 +455,153 @@ class ComBatModel:
|
|
|
431
455
|
self._pc_gamma_star = gamma_star
|
|
432
456
|
self._pc_delta_star = delta_star
|
|
433
457
|
|
|
458
|
+
def _build_longitudinal_design(
|
|
459
|
+
self,
|
|
460
|
+
X: pd.DataFrame,
|
|
461
|
+
batch: pd.Series,
|
|
462
|
+
disc: pd.DataFrame | None,
|
|
463
|
+
cont: pd.DataFrame | None,
|
|
464
|
+
time: pd.DataFrame | None,
|
|
465
|
+
) -> tuple[FloatArray, list[str]]:
|
|
466
|
+
"""Assemble the fixed-effects design [batch dummies | covariates | time].
|
|
467
|
+
|
|
468
|
+
Returns the design matrix and the list of non-batch column names.
|
|
469
|
+
"""
|
|
470
|
+
batch_dummies = pd.get_dummies(batch, drop_first=False).astype(float)
|
|
471
|
+
batch_dummies = batch_dummies.reindex(columns=self._batch_levels, fill_value=0.0)
|
|
472
|
+
|
|
473
|
+
nonbatch_parts: list[pd.DataFrame] = []
|
|
474
|
+
if disc is not None:
|
|
475
|
+
nonbatch_parts.append(
|
|
476
|
+
pd.get_dummies(disc.astype("category"), drop_first=True).astype(float)
|
|
477
|
+
)
|
|
478
|
+
if cont is not None:
|
|
479
|
+
nonbatch_parts.append(cont.astype(float))
|
|
480
|
+
if time is not None:
|
|
481
|
+
nonbatch_parts.append(time.astype(float))
|
|
482
|
+
|
|
483
|
+
if nonbatch_parts:
|
|
484
|
+
nonbatch_df = pd.concat(nonbatch_parts, axis=1)
|
|
485
|
+
else:
|
|
486
|
+
nonbatch_df = pd.DataFrame(index=batch_dummies.index)
|
|
487
|
+
|
|
488
|
+
design_df = pd.concat([batch_dummies, nonbatch_df], axis=1)
|
|
489
|
+
n_batch = len(self._batch_levels)
|
|
490
|
+
nonbatch_columns = design_df.columns[n_batch:].tolist()
|
|
491
|
+
return design_df.values.astype(np.float64), nonbatch_columns
|
|
492
|
+
|
|
493
|
+
def _fit_longitudinal(
|
|
494
|
+
self,
|
|
495
|
+
X: pd.DataFrame,
|
|
496
|
+
batch: pd.Series,
|
|
497
|
+
disc: pd.DataFrame | None,
|
|
498
|
+
cont: pd.DataFrame | None,
|
|
499
|
+
subject_id: ArrayLike | None,
|
|
500
|
+
time_covariate: ArrayLike | None,
|
|
501
|
+
) -> None:
|
|
502
|
+
"""Longitudinal ComBat (*Beer et al.* 2020).
|
|
503
|
+
|
|
504
|
+
Identical to Fortin's standardize -> per-batch gamma/delta -> empirical
|
|
505
|
+
Bayes -> remove pipeline, except the fixed-effects mean model is fit with
|
|
506
|
+
a per-subject random intercept (REML), and the random-intercept BLUP is
|
|
507
|
+
included in the standardization mean.
|
|
508
|
+
"""
|
|
509
|
+
if subject_id is None:
|
|
510
|
+
raise ValueError(
|
|
511
|
+
"method='longitudinal' requires `subject_id` (one label per sample "
|
|
512
|
+
"identifying the subject/individual used for the random intercept)."
|
|
513
|
+
)
|
|
514
|
+
|
|
515
|
+
self._batch_levels = batch.cat.categories
|
|
516
|
+
n_batch = len(self._batch_levels)
|
|
517
|
+
n_samples = len(X)
|
|
518
|
+
|
|
519
|
+
counts = batch.value_counts()
|
|
520
|
+
small = counts[counts < 2]
|
|
521
|
+
if len(small):
|
|
522
|
+
lvl = small.index[0]
|
|
523
|
+
raise ValueError(
|
|
524
|
+
f"Batch '{lvl}' has only {int(small.iloc[0])} sample(s). ComBat requires "
|
|
525
|
+
f">= 2 samples per batch. Consider merging small batches or removing them."
|
|
526
|
+
)
|
|
527
|
+
|
|
528
|
+
time = self._to_df(time_covariate, X.index, "time_covariate")
|
|
529
|
+
design, self._nonbatch_columns = self._build_longitudinal_design(X, batch, disc, cont, time)
|
|
530
|
+
self._design_cond = float(la.cond(design)) if design.shape[1] else 1.0
|
|
531
|
+
self._n_batch = n_batch
|
|
532
|
+
self._p_design = design.shape[1]
|
|
533
|
+
|
|
534
|
+
subj = self._as_series(subject_id, X.index, "subject_id")
|
|
535
|
+
self._re_subject_levels = subj.cat.categories
|
|
536
|
+
group_idx = subj.cat.codes.to_numpy().astype(np.intp)
|
|
537
|
+
n_groups = len(self._re_subject_levels)
|
|
538
|
+
n_k = np.bincount(group_idx, minlength=n_groups).astype(np.float64)
|
|
539
|
+
|
|
540
|
+
beta, blup, sigma2, _ = fit_random_intercept(
|
|
541
|
+
design, X.values.astype(np.float64), group_idx, n_groups, n_k, eps=self.eps
|
|
542
|
+
)
|
|
543
|
+
self._re_blup = blup
|
|
544
|
+
|
|
545
|
+
beta_batch = beta[:n_batch]
|
|
546
|
+
self._beta_hat_nonbatch = beta[n_batch:]
|
|
547
|
+
|
|
548
|
+
n_per_batch_arr = np.asarray(
|
|
549
|
+
batch.value_counts().reindex(self._batch_levels).astype(int).values, dtype=np.float64
|
|
550
|
+
)
|
|
551
|
+
self._n_per_batch = dict(zip(self._batch_levels, n_per_batch_arr.astype(int), strict=True))
|
|
552
|
+
|
|
553
|
+
grand_mean = (n_per_batch_arr / n_samples) @ beta_batch
|
|
554
|
+
self._grand_mean = pd.Series(grand_mean, index=X.columns)
|
|
555
|
+
var_pooled = sigma2 + self.eps
|
|
556
|
+
self._pooled_var = pd.Series(var_pooled, index=X.columns)
|
|
557
|
+
|
|
558
|
+
blup_per_row = blup[group_idx]
|
|
559
|
+
stand_mean = grand_mean + design[:, n_batch:] @ self._beta_hat_nonbatch + blup_per_row
|
|
560
|
+
Xs = (X.values - stand_mean) / np.sqrt(var_pooled)
|
|
561
|
+
|
|
562
|
+
gamma_hat = np.vstack(
|
|
563
|
+
[Xs[(batch == lvl).to_numpy()].mean(axis=0) for lvl in self._batch_levels]
|
|
564
|
+
)
|
|
565
|
+
delta_hat = np.vstack(
|
|
566
|
+
[
|
|
567
|
+
Xs[(batch == lvl).to_numpy()].var(axis=0, ddof=1) + self.eps
|
|
568
|
+
for lvl in self._batch_levels
|
|
569
|
+
]
|
|
570
|
+
)
|
|
571
|
+
|
|
572
|
+
b_names = [str(lvl) for lvl in self._batch_levels]
|
|
573
|
+
if self.mean_only:
|
|
574
|
+
gamma_star = self._shrink_gamma(
|
|
575
|
+
gamma_hat,
|
|
576
|
+
delta_hat,
|
|
577
|
+
n_per_batch_arr,
|
|
578
|
+
parametric=self.parametric,
|
|
579
|
+
batch_names=b_names,
|
|
580
|
+
)
|
|
581
|
+
delta_star = np.ones_like(delta_hat)
|
|
582
|
+
else:
|
|
583
|
+
gamma_star, delta_star = self._shrink_gamma_delta(
|
|
584
|
+
gamma_hat,
|
|
585
|
+
delta_hat,
|
|
586
|
+
n_per_batch_arr,
|
|
587
|
+
parametric=self.parametric,
|
|
588
|
+
batch_names=b_names,
|
|
589
|
+
)
|
|
590
|
+
|
|
591
|
+
if self.reference_batch is not None:
|
|
592
|
+
ref_idx = list(self._batch_levels).index(self.reference_batch)
|
|
593
|
+
gamma_ref = gamma_star[ref_idx]
|
|
594
|
+
delta_ref = delta_star[ref_idx]
|
|
595
|
+
gamma_star = gamma_star - gamma_ref
|
|
596
|
+
if not self.mean_only:
|
|
597
|
+
delta_star = delta_star / delta_ref
|
|
598
|
+
self._reference_batch_idx = ref_idx
|
|
599
|
+
else:
|
|
600
|
+
self._reference_batch_idx = None
|
|
601
|
+
|
|
602
|
+
self._gamma_star = gamma_star
|
|
603
|
+
self._delta_star = delta_star
|
|
604
|
+
|
|
434
605
|
def _shrink_gamma_delta(
|
|
435
606
|
self,
|
|
436
607
|
gamma_hat: FloatArray,
|
|
@@ -620,6 +791,8 @@ class ComBatModel:
|
|
|
620
791
|
batch: ArrayLike,
|
|
621
792
|
discrete_covariates: ArrayLike | None = None,
|
|
622
793
|
continuous_covariates: ArrayLike | None = None,
|
|
794
|
+
subject_id: ArrayLike | None = None,
|
|
795
|
+
time_covariate: ArrayLike | None = None,
|
|
623
796
|
) -> pd.DataFrame:
|
|
624
797
|
"""Transform the data using fitted ComBat parameters.
|
|
625
798
|
|
|
@@ -630,9 +803,14 @@ class ComBatModel:
|
|
|
630
803
|
batch : array-like of shape (n_samples,)
|
|
631
804
|
Batch labels for each sample.
|
|
632
805
|
discrete_covariates : array-like or None, default=None
|
|
633
|
-
Categorical covariates (Fortin/Chen only).
|
|
806
|
+
Categorical covariates (Fortin/Chen/Longitudinal only).
|
|
634
807
|
continuous_covariates : array-like or None, default=None
|
|
635
|
-
Continuous covariates (Fortin/Chen only).
|
|
808
|
+
Continuous covariates (Fortin/Chen/Longitudinal only).
|
|
809
|
+
subject_id : array-like or None, default=None
|
|
810
|
+
Subject labels for the random intercept (Longitudinal only). Unseen
|
|
811
|
+
subjects fall back to a zero random intercept.
|
|
812
|
+
time_covariate : array-like or None, default=None
|
|
813
|
+
Continuous time variable (Longitudinal only).
|
|
636
814
|
|
|
637
815
|
Returns
|
|
638
816
|
-------
|
|
@@ -665,9 +843,12 @@ class ComBatModel:
|
|
|
665
843
|
return self._transform_fortin(X, batch, disc, cont)
|
|
666
844
|
elif method == "chen":
|
|
667
845
|
return self._transform_chen(X, batch, disc, cont)
|
|
846
|
+
elif method == "longitudinal":
|
|
847
|
+
return self._transform_longitudinal(X, batch, disc, cont, subject_id, time_covariate)
|
|
668
848
|
else:
|
|
669
849
|
raise ValueError(
|
|
670
|
-
f"Unknown method={method!r}. Expected one of 'johnson', 'fortin',
|
|
850
|
+
f"Unknown method={method!r}. Expected one of 'johnson', 'fortin', "
|
|
851
|
+
f"'chen', or 'longitudinal'."
|
|
671
852
|
)
|
|
672
853
|
|
|
673
854
|
def _transform_johnson(self, X: pd.DataFrame, batch: pd.Series) -> pd.DataFrame:
|
|
@@ -773,3 +954,55 @@ class ComBatModel:
|
|
|
773
954
|
|
|
774
955
|
X_recon = self._covbat_pca.inverse_transform(scores_adj)
|
|
775
956
|
return pd.DataFrame(X_recon, index=X.index, columns=X.columns)
|
|
957
|
+
|
|
958
|
+
def _transform_longitudinal(
|
|
959
|
+
self,
|
|
960
|
+
X: pd.DataFrame,
|
|
961
|
+
batch: pd.Series,
|
|
962
|
+
disc: pd.DataFrame | None,
|
|
963
|
+
cont: pd.DataFrame | None,
|
|
964
|
+
subject_id: ArrayLike | None,
|
|
965
|
+
time_covariate: ArrayLike | None,
|
|
966
|
+
) -> pd.DataFrame:
|
|
967
|
+
"""Longitudinal transform implementation."""
|
|
968
|
+
if subject_id is None:
|
|
969
|
+
raise ValueError("method='longitudinal' requires `subject_id` at transform time.")
|
|
970
|
+
|
|
971
|
+
n_features = len(self._grand_mean)
|
|
972
|
+
time = self._to_df(time_covariate, X.index, "time_covariate")
|
|
973
|
+
design, nonbatch_columns = self._build_longitudinal_design(X, batch, disc, cont, time)
|
|
974
|
+
# Realign non-batch columns to those seen at fit (handles unseen covariate levels).
|
|
975
|
+
nonbatch_design = pd.DataFrame(
|
|
976
|
+
design[:, self._n_batch :],
|
|
977
|
+
index=X.index,
|
|
978
|
+
columns=nonbatch_columns,
|
|
979
|
+
).reindex(columns=self._nonbatch_columns, fill_value=0.0)
|
|
980
|
+
|
|
981
|
+
subj = self._as_series(subject_id, X.index, "subject_id")
|
|
982
|
+
level_pos = {lvl: i for i, lvl in enumerate(self._re_subject_levels)}
|
|
983
|
+
codes = np.array([level_pos.get(s, -1) for s in subj.to_numpy()], dtype=np.intp)
|
|
984
|
+
blup_per_row = np.zeros((len(X.index), n_features), dtype=np.float64)
|
|
985
|
+
seen = codes >= 0
|
|
986
|
+
if seen.any():
|
|
987
|
+
blup_per_row[seen] = self._re_blup[codes[seen]]
|
|
988
|
+
|
|
989
|
+
grand_vals = np.asarray(self._grand_mean.values, dtype=np.float64)
|
|
990
|
+
pooled_vals = np.asarray(self._pooled_var.values, dtype=np.float64)
|
|
991
|
+
stand_mean = grand_vals + nonbatch_design.values @ self._beta_hat_nonbatch + blup_per_row
|
|
992
|
+
Xs = (X.values - stand_mean) / np.sqrt(pooled_vals)
|
|
993
|
+
|
|
994
|
+
for i, lvl in enumerate(self._batch_levels):
|
|
995
|
+
idx = (batch == lvl).to_numpy()
|
|
996
|
+
if not idx.any():
|
|
997
|
+
continue
|
|
998
|
+
if self.reference_batch is not None and lvl == self.reference_batch:
|
|
999
|
+
continue
|
|
1000
|
+
g = self._gamma_star[i]
|
|
1001
|
+
d = self._delta_star[i]
|
|
1002
|
+
if self.mean_only:
|
|
1003
|
+
Xs[idx] = Xs[idx] - g
|
|
1004
|
+
else:
|
|
1005
|
+
Xs[idx] = (Xs[idx] - g) / np.sqrt(d)
|
|
1006
|
+
|
|
1007
|
+
X_adj = Xs * np.sqrt(pooled_vals) + stand_mean
|
|
1008
|
+
return pd.DataFrame(X_adj, index=X.index, columns=X.columns, dtype=float)
|
|
@@ -27,10 +27,15 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
|
|
|
27
27
|
batch : array-like of shape (n_samples,)
|
|
28
28
|
Batch labels for each sample.
|
|
29
29
|
discrete_covariates : array-like, optional
|
|
30
|
-
Categorical covariates to protect (Fortin/Chen only).
|
|
30
|
+
Categorical covariates to protect (Fortin/Chen/Longitudinal only).
|
|
31
31
|
continuous_covariates : array-like, optional
|
|
32
|
-
Continuous covariates to protect (Fortin/Chen only).
|
|
33
|
-
|
|
32
|
+
Continuous covariates to protect (Fortin/Chen/Longitudinal only).
|
|
33
|
+
subject_id : array-like, optional
|
|
34
|
+
Subject/individual labels for the random intercept. Required for
|
|
35
|
+
``method='longitudinal'``, ignored otherwise.
|
|
36
|
+
time_covariate : array-like, optional
|
|
37
|
+
Continuous time variable for repeated measures (Longitudinal only).
|
|
38
|
+
method : {'johnson', 'fortin', 'chen', 'longitudinal'}, default='johnson'
|
|
34
39
|
ComBat variant to use.
|
|
35
40
|
parametric : bool, default=True
|
|
36
41
|
Use parametric empirical Bayes.
|
|
@@ -53,6 +58,8 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
|
|
|
53
58
|
*,
|
|
54
59
|
discrete_covariates: ArrayLike | None = None,
|
|
55
60
|
continuous_covariates: ArrayLike | None = None,
|
|
61
|
+
subject_id: ArrayLike | None = None,
|
|
62
|
+
time_covariate: ArrayLike | None = None,
|
|
56
63
|
method: str = "johnson",
|
|
57
64
|
parametric: bool = True,
|
|
58
65
|
mean_only: bool = False,
|
|
@@ -63,6 +70,8 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
|
|
|
63
70
|
self.batch = batch
|
|
64
71
|
self.discrete_covariates = discrete_covariates
|
|
65
72
|
self.continuous_covariates = continuous_covariates
|
|
73
|
+
self.subject_id = subject_id
|
|
74
|
+
self.time_covariate = time_covariate
|
|
66
75
|
self.method = method
|
|
67
76
|
self.parametric = parametric
|
|
68
77
|
self.mean_only = mean_only
|
|
@@ -85,6 +94,19 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
|
|
|
85
94
|
f"All batch labels must be non-null. Check your data for missing entries."
|
|
86
95
|
)
|
|
87
96
|
|
|
97
|
+
if self.subject_id is not None:
|
|
98
|
+
subj_ser = (
|
|
99
|
+
pd.Series(self.subject_id)
|
|
100
|
+
if not isinstance(self.subject_id, pd.Series)
|
|
101
|
+
else self.subject_id
|
|
102
|
+
)
|
|
103
|
+
nan_count = int(subj_ser.isna().sum())
|
|
104
|
+
if nan_count:
|
|
105
|
+
raise ValueError(
|
|
106
|
+
f"subject_id contains {nan_count} NaN value(s). "
|
|
107
|
+
f"All subject labels must be non-null."
|
|
108
|
+
)
|
|
109
|
+
|
|
88
110
|
if self.discrete_covariates is not None:
|
|
89
111
|
disc_df = (
|
|
90
112
|
pd.DataFrame(self.discrete_covariates)
|
|
@@ -118,6 +140,7 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
|
|
|
118
140
|
batch_ser: pd.Series,
|
|
119
141
|
disc: pd.DataFrame | pd.Series | None,
|
|
120
142
|
cont: pd.DataFrame | pd.Series | None,
|
|
143
|
+
time: pd.DataFrame | pd.Series | None = None,
|
|
121
144
|
) -> None:
|
|
122
145
|
"""Issue warnings for data quality issues that may affect results."""
|
|
123
146
|
# Near-zero variance features
|
|
@@ -148,8 +171,10 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
|
|
|
148
171
|
stacklevel=3,
|
|
149
172
|
)
|
|
150
173
|
|
|
151
|
-
# Covariate collinearity with batch (fortin/chen only)
|
|
152
|
-
if self.method.lower() in {"fortin", "chen"} and (
|
|
174
|
+
# Covariate collinearity with batch (fortin/chen/longitudinal only)
|
|
175
|
+
if self.method.lower() in {"fortin", "chen", "longitudinal"} and (
|
|
176
|
+
disc is not None or cont is not None or time is not None
|
|
177
|
+
):
|
|
153
178
|
batch_dummies = pd.get_dummies(batch_ser, drop_first=True).astype(float)
|
|
154
179
|
cov_parts: list[pd.DataFrame] = []
|
|
155
180
|
if disc is not None:
|
|
@@ -160,6 +185,9 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
|
|
|
160
185
|
if cont is not None:
|
|
161
186
|
c = cont if isinstance(cont, pd.DataFrame) else cont.to_frame()
|
|
162
187
|
cov_parts.append(c.astype(float))
|
|
188
|
+
if time is not None:
|
|
189
|
+
t = time if isinstance(time, pd.DataFrame) else time.to_frame()
|
|
190
|
+
cov_parts.append(t.astype(float))
|
|
163
191
|
cov_df = pd.concat(cov_parts, axis=1)
|
|
164
192
|
design = pd.concat([batch_dummies, cov_df], axis=1)
|
|
165
193
|
rank = la.matrix_rank(design.values)
|
|
@@ -227,14 +255,18 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
|
|
|
227
255
|
batch_vec = _subset(self.batch, idx)
|
|
228
256
|
disc = _subset(self.discrete_covariates, idx)
|
|
229
257
|
cont = _subset(self.continuous_covariates, idx)
|
|
258
|
+
subj = _subset(self.subject_id, idx)
|
|
259
|
+
time = _subset(self.time_covariate, idx)
|
|
230
260
|
|
|
231
|
-
self._check_data_quality(X_df, batch_vec, disc, cont) # type: ignore[arg-type]
|
|
261
|
+
self._check_data_quality(X_df, batch_vec, disc, cont, time) # type: ignore[arg-type]
|
|
232
262
|
|
|
233
263
|
self._model.fit(
|
|
234
264
|
X,
|
|
235
265
|
batch=batch_vec, # type: ignore[arg-type]
|
|
236
266
|
discrete_covariates=disc,
|
|
237
267
|
continuous_covariates=cont,
|
|
268
|
+
subject_id=subj,
|
|
269
|
+
time_covariate=time,
|
|
238
270
|
)
|
|
239
271
|
self._fitted_batch = batch_vec
|
|
240
272
|
|
|
@@ -261,11 +293,15 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
|
|
|
261
293
|
batch_vec = _subset(self.batch, idx)
|
|
262
294
|
disc = _subset(self.discrete_covariates, idx)
|
|
263
295
|
cont = _subset(self.continuous_covariates, idx)
|
|
296
|
+
subj = _subset(self.subject_id, idx)
|
|
297
|
+
time = _subset(self.time_covariate, idx)
|
|
264
298
|
X_transformed = self._model.transform(
|
|
265
299
|
X,
|
|
266
300
|
batch=batch_vec, # type: ignore[arg-type]
|
|
267
301
|
discrete_covariates=disc,
|
|
268
302
|
continuous_covariates=cont,
|
|
303
|
+
subject_id=subj,
|
|
304
|
+
time_covariate=time,
|
|
269
305
|
)
|
|
270
306
|
|
|
271
307
|
batch_arr = np.asarray(batch_vec)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: combatlearn
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: Batch-effect harmonization for machine learning frameworks.
|
|
5
5
|
Author-email: Ettore Rocchi <ettoreroc@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -27,6 +27,7 @@ Requires-Dist: ruff>=0.8; extra == "dev"
|
|
|
27
27
|
Requires-Dist: pre-commit>=3.0; extra == "dev"
|
|
28
28
|
Requires-Dist: mypy>=1.0; extra == "dev"
|
|
29
29
|
Requires-Dist: pandas-stubs; extra == "dev"
|
|
30
|
+
Requires-Dist: statsmodels>=0.14; extra == "dev"
|
|
30
31
|
Provides-Extra: docs
|
|
31
32
|
Requires-Dist: sphinx>=7.0.0; extra == "docs"
|
|
32
33
|
Requires-Dist: pydata-sphinx-theme>=0.15.0; extra == "docs"
|
|
@@ -49,10 +50,11 @@ Dynamic: license-file
|
|
|
49
50
|
|
|
50
51
|
**combatlearn** makes the popular _ComBat_ (and _CovBat_) batch-effect correction algorithm available for use into machine learning frameworks. It lets you harmonise high-dimensional data inside a scikit-learn `Pipeline`, so that cross-validation and grid-search automatically take batch structure into account, **without data leakage**.
|
|
51
52
|
|
|
52
|
-
**
|
|
53
|
+
**Four methods**:
|
|
53
54
|
- `method="johnson"` - classic ComBat (Johnson _et al._, 2007)
|
|
54
55
|
- `method="fortin"` - neuroComBat (Fortin _et al._, 2018)
|
|
55
56
|
- `method="chen"` - CovBat (Chen _et al._, 2022)
|
|
57
|
+
- `method="longitudinal"` - Longitudinal ComBat for repeated measures (Beer _et al._, 2020)
|
|
56
58
|
|
|
57
59
|
## Installation
|
|
58
60
|
|
|
@@ -75,6 +77,7 @@ import pandas as pd
|
|
|
75
77
|
from sklearn.pipeline import Pipeline
|
|
76
78
|
from sklearn.preprocessing import StandardScaler
|
|
77
79
|
from sklearn.linear_model import LogisticRegression
|
|
80
|
+
from sklearn.model_selection import GridSearchCV
|
|
78
81
|
from combatlearn import ComBat
|
|
79
82
|
|
|
80
83
|
df = pd.read_csv("data.csv", index_col=0)
|
|
@@ -125,18 +128,20 @@ The following section provides a detailed explanation of all parameters availabl
|
|
|
125
128
|
| Parameter | Type | Default | Description |
|
|
126
129
|
| --- | --- | --- | --- |
|
|
127
130
|
| `batch` | array-like or pd.Series | **required** | Vector indicating batch assignment for each sample. This is used to estimate and remove batch effects. |
|
|
128
|
-
| `discrete_covariates` | array-like, pd.Series, or pd.DataFrame | `None` | Optional categorical covariates (e.g., sex, site). Only used in `"fortin"` and `"
|
|
129
|
-
| `continuous_covariates` | array-like, pd.Series or pd.DataFrame | `None` | Optional continuous covariates (e.g., age). Only used in `"fortin"` and `"
|
|
131
|
+
| `discrete_covariates` | array-like, pd.Series, or pd.DataFrame | `None` | Optional categorical covariates (e.g., sex, site). Only used in `"fortin"`, `"chen"`, and `"longitudinal"` methods. |
|
|
132
|
+
| `continuous_covariates` | array-like, pd.Series or pd.DataFrame | `None` | Optional continuous covariates (e.g., age). Only used in `"fortin"`, `"chen"`, and `"longitudinal"` methods. |
|
|
133
|
+
| `subject_id` | array-like or pd.Series | `None` | Subject/individual labels for the random intercept. **Required** for `method="longitudinal"`, ignored otherwise. |
|
|
134
|
+
| `time_covariate` | array-like or pd.Series | `None` | Optional continuous time variable for repeated measures. Only used in `"longitudinal"`. |
|
|
130
135
|
|
|
131
136
|
### Algorithm Options
|
|
132
137
|
|
|
133
138
|
| Parameter | Type | Default | Description |
|
|
134
139
|
| --- | --- | --- | --- |
|
|
135
|
-
| `method` | str | `"johnson"` | ComBat method to use: <ul><li>`"johnson"` - Classical ComBat (_Johnson et al. 2007_)</li><li>`"fortin"` - ComBat with covariates (_Fortin et al. 2018_)</li><li>`"chen"` - CovBat, PCA-based correction (_Chen et al. 2022_)</li></ul> |
|
|
140
|
+
| `method` | str | `"johnson"` | ComBat method to use: <ul><li>`"johnson"` - Classical ComBat (_Johnson et al. 2007_)</li><li>`"fortin"` - ComBat with covariates (_Fortin et al. 2018_)</li><li>`"chen"` - CovBat, PCA-based correction (_Chen et al. 2022_)</li><li>`"longitudinal"` - Longitudinal ComBat with a per-subject random intercept (_Beer et al. 2020_)</li></ul> |
|
|
136
141
|
| `parametric` | bool | `True` | Whether to use the **parametric empirical Bayes** formulation. If `False`, a non-parametric iterative scheme is used. |
|
|
137
142
|
| `mean_only` | bool | `False` | If `True`, only the **mean** is corrected, while variances are left unchanged. Useful for preserving variance structure in the data. |
|
|
138
143
|
| `reference_batch` | str or `None` | `None` | If specified, acts as a reference batch - other batches will be corrected to match this one. |
|
|
139
|
-
| `covbat_cov_thresh` | float, int | `0.9` | For `"chen"` method only: Cumulative variance threshold
|
|
144
|
+
| `covbat_cov_thresh` | float, int | `0.9` | For `"chen"` method only: Cumulative variance threshold in `(0, 1]` to retain PCs in PCA space (e.g., 0.9 = retain 90% explained variance). If an integer is provided, it represents the number of principal components to use. |
|
|
140
145
|
| `eps` | float | `1e-8` | Small jitter value added to variances to prevent divide-by-zero errors during standardization. |
|
|
141
146
|
|
|
142
147
|
|
|
@@ -190,3 +195,5 @@ Please consider citing the original papers:
|
|
|
190
195
|
- [**neuroCombat**](https://github.com/Jfortin1/neuroCombat) - Fortin JP et al. _Neuroimage_. 2018. doi: [10.1016/j.neuroimage.2017.11.024](https://doi.org/10.1016/j.neuroimage.2017.11.024)
|
|
191
196
|
|
|
192
197
|
- [**CovBat**](https://github.com/andy1764/CovBat_Harmonization) - Chen AA et al. _Hum Brain Mapp_. 2022. doi: [10.1002/hbm.25688](https://doi.org/10.1002/hbm.25688)
|
|
198
|
+
|
|
199
|
+
- [**Longitudinal ComBat**](https://github.com/jcbeer/longCombat) - Beer JC et al. _Neuroimage_. 2020. doi: [10.1016/j.neuroimage.2020.117129](https://doi.org/10.1016/j.neuroimage.2020.117129)
|
|
@@ -5,6 +5,7 @@ requirements-dev.txt
|
|
|
5
5
|
requirements-docs.txt
|
|
6
6
|
requirements.txt
|
|
7
7
|
combatlearn/__init__.py
|
|
8
|
+
combatlearn/_mixed.py
|
|
8
9
|
combatlearn/_utils.py
|
|
9
10
|
combatlearn/core.py
|
|
10
11
|
combatlearn/inspection.py
|
|
@@ -19,5 +20,6 @@ combatlearn.egg-info/requires.txt
|
|
|
19
20
|
combatlearn.egg-info/top_level.txt
|
|
20
21
|
tests/test_combat.py
|
|
21
22
|
tests/test_inspection.py
|
|
23
|
+
tests/test_longitudinal.py
|
|
22
24
|
tests/test_metrics.py
|
|
23
25
|
tests/test_visualization.py
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import pytest
|
|
4
|
+
from sklearn.base import clone
|
|
5
|
+
from sklearn.pipeline import Pipeline
|
|
6
|
+
from sklearn.preprocessing import StandardScaler
|
|
7
|
+
from utils import simulate_longitudinal_data
|
|
8
|
+
|
|
9
|
+
from combatlearn import ComBat
|
|
10
|
+
from combatlearn._mixed import fit_random_intercept
|
|
11
|
+
from combatlearn.core import ComBatModel
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _fit_kwargs():
|
|
15
|
+
X, batch, subject, time = simulate_longitudinal_data()
|
|
16
|
+
return X, {
|
|
17
|
+
"batch": batch,
|
|
18
|
+
"subject_id": subject,
|
|
19
|
+
"time_covariate": time,
|
|
20
|
+
"method": "longitudinal",
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_longitudinal_shape_index_columns_preserved():
|
|
25
|
+
"""Output keeps shape, index, and column names."""
|
|
26
|
+
X, kwargs = _fit_kwargs()
|
|
27
|
+
X.index = pd.Index([f"obs_{i}" for i in range(len(X))])
|
|
28
|
+
kwargs["batch"].index = X.index
|
|
29
|
+
kwargs["subject_id"].index = X.index
|
|
30
|
+
kwargs["time_covariate"].index = X.index
|
|
31
|
+
X.columns = [f"feat_{i}" for i in range(X.shape[1])]
|
|
32
|
+
|
|
33
|
+
X_corr = ComBat(**kwargs).fit_transform(X)
|
|
34
|
+
assert X_corr.shape == X.shape
|
|
35
|
+
assert X_corr.index.equals(X.index)
|
|
36
|
+
assert list(X_corr.columns) == list(X.columns)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_longitudinal_no_nan_or_inf():
|
|
40
|
+
"""Correction introduces no NaN/Inf."""
|
|
41
|
+
X, kwargs = _fit_kwargs()
|
|
42
|
+
X_corr = ComBat(**kwargs).fit_transform(X)
|
|
43
|
+
assert not np.isnan(X_corr.values).any()
|
|
44
|
+
assert not np.isinf(X_corr.values).any()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@pytest.mark.parametrize("mean_only", [True, False])
|
|
48
|
+
def test_longitudinal_mean_only(mean_only):
|
|
49
|
+
"""mean_only mode works."""
|
|
50
|
+
X, kwargs = _fit_kwargs()
|
|
51
|
+
X_corr = ComBat(mean_only=mean_only, **kwargs).fit_transform(X)
|
|
52
|
+
assert X_corr.shape == X.shape
|
|
53
|
+
assert not np.isnan(X_corr.values).any()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@pytest.mark.parametrize("parametric", [True, False])
|
|
57
|
+
def test_longitudinal_parametric_flag(parametric):
|
|
58
|
+
"""Both EB variants run."""
|
|
59
|
+
X, kwargs = _fit_kwargs()
|
|
60
|
+
X_corr = ComBat(parametric=parametric, **kwargs).fit_transform(X)
|
|
61
|
+
assert X_corr.shape == X.shape
|
|
62
|
+
assert not np.isnan(X_corr.values).any()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def test_longitudinal_reduces_batch_variance():
|
|
66
|
+
"""Harmonization should reduce the fraction of variance explained by batch."""
|
|
67
|
+
X, kwargs = _fit_kwargs()
|
|
68
|
+
cb = ComBat(**kwargs).fit(X)
|
|
69
|
+
cb.transform(X)
|
|
70
|
+
assert cb._batch_var_after_ < cb._batch_var_before_
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_longitudinal_requires_subject_id():
|
|
74
|
+
"""method='longitudinal' without subject_id raises."""
|
|
75
|
+
X, batch, _subject, time = simulate_longitudinal_data()
|
|
76
|
+
with pytest.raises(ValueError, match=r"requires `subject_id`"):
|
|
77
|
+
ComBat(batch=batch, time_covariate=time, method="longitudinal").fit(X)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_subject_id_ignored_warning_for_other_methods():
|
|
81
|
+
"""Passing subject_id with a non-longitudinal method warns that it is ignored."""
|
|
82
|
+
X, batch, subject, _time = simulate_longitudinal_data()
|
|
83
|
+
with pytest.warns(UserWarning, match="only used when"):
|
|
84
|
+
ComBat(batch=batch, subject_id=subject, method="johnson").fit(X)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_longitudinal_reference_batch_unchanged():
|
|
88
|
+
"""Reference-batch samples come out numerically identical."""
|
|
89
|
+
X, batch, subject, time = simulate_longitudinal_data()
|
|
90
|
+
ref = batch.iloc[0]
|
|
91
|
+
cb = ComBat(
|
|
92
|
+
batch=batch,
|
|
93
|
+
subject_id=subject,
|
|
94
|
+
time_covariate=time,
|
|
95
|
+
method="longitudinal",
|
|
96
|
+
reference_batch=ref,
|
|
97
|
+
).fit(X)
|
|
98
|
+
X_corr = cb.transform(X)
|
|
99
|
+
mask = (batch == ref).to_numpy()
|
|
100
|
+
np.testing.assert_allclose(X_corr.values[mask], X.values[mask], rtol=0, atol=1e-8)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def test_longitudinal_clone_and_pipeline():
|
|
104
|
+
"""Wrapper clones and runs inside a Pipeline reproducibly."""
|
|
105
|
+
X, kwargs = _fit_kwargs()
|
|
106
|
+
pipe = Pipeline([("combat", ComBat(**kwargs)), ("scaler", StandardScaler())])
|
|
107
|
+
X_corr = pipe.fit_transform(X)
|
|
108
|
+
X_corr2 = clone(pipe).fit_transform(X)
|
|
109
|
+
np.testing.assert_allclose(X_corr, X_corr2, rtol=1e-6, atol=1e-6)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def test_longitudinal_unseen_subject_falls_back_to_zero_re():
|
|
113
|
+
"""Transforming subjects unseen at fit (zero random intercept) must not crash."""
|
|
114
|
+
X, batch, subject, time = simulate_longitudinal_data()
|
|
115
|
+
model = ComBatModel(method="longitudinal").fit(
|
|
116
|
+
X, batch=batch, subject_id=subject, time_covariate=time
|
|
117
|
+
)
|
|
118
|
+
subject_new = subject.map(lambda s: f"new_{s}")
|
|
119
|
+
X_corr = model.transform(X, batch=batch, subject_id=subject_new, time_covariate=time)
|
|
120
|
+
assert X_corr.shape == X.shape
|
|
121
|
+
assert not np.isnan(X_corr.values).any()
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def test_longitudinal_no_covariates():
|
|
125
|
+
"""Longitudinal with only a subject random intercept (no covariates) works."""
|
|
126
|
+
X, batch, subject, _time = simulate_longitudinal_data()
|
|
127
|
+
X_corr = ComBat(batch=batch, subject_id=subject, method="longitudinal").fit_transform(X)
|
|
128
|
+
assert X_corr.shape == X.shape
|
|
129
|
+
assert not np.isnan(X_corr.values).any()
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def test_longitudinal_collinear_discrete_covariate_warns():
|
|
133
|
+
"""A discrete covariate collinear with batch should warn for the longitudinal method too."""
|
|
134
|
+
X, batch, subject, _time = simulate_longitudinal_data()
|
|
135
|
+
disc_collinear = batch.to_frame("cov")
|
|
136
|
+
with pytest.warns(UserWarning, match="rank-deficient"):
|
|
137
|
+
ComBat(
|
|
138
|
+
batch=batch,
|
|
139
|
+
subject_id=subject,
|
|
140
|
+
discrete_covariates=disc_collinear,
|
|
141
|
+
method="longitudinal",
|
|
142
|
+
).fit(X)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def test_longitudinal_collinear_time_covariate_warns():
|
|
146
|
+
"""A time covariate collinear with batch should also trigger the rank-deficiency warning."""
|
|
147
|
+
X, batch, subject, _time = simulate_longitudinal_data()
|
|
148
|
+
time_collinear = batch.map({"A": 0.0, "B": 1.0, "C": 2.0}).rename("visit")
|
|
149
|
+
with pytest.warns(UserWarning, match="rank-deficient"):
|
|
150
|
+
ComBat(
|
|
151
|
+
batch=batch,
|
|
152
|
+
subject_id=subject,
|
|
153
|
+
time_covariate=time_collinear,
|
|
154
|
+
method="longitudinal",
|
|
155
|
+
).fit(X)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def test_random_intercept_recovers_fixed_effects():
|
|
159
|
+
"""The REML fitter recovers known fixed effects on simulated data."""
|
|
160
|
+
rng = np.random.default_rng(0)
|
|
161
|
+
n_subjects, n_times = 60, 4
|
|
162
|
+
n = n_subjects * n_times
|
|
163
|
+
group_idx = np.repeat(np.arange(n_subjects), n_times).astype(np.intp)
|
|
164
|
+
n_k = np.bincount(group_idx).astype(np.float64)
|
|
165
|
+
|
|
166
|
+
x1 = rng.standard_normal(n)
|
|
167
|
+
beta_true = np.array([2.0, 1.5])
|
|
168
|
+
u = rng.normal(0.0, 1.0, n_subjects)
|
|
169
|
+
eps = rng.normal(0.0, 1.0, n)
|
|
170
|
+
y = beta_true[0] + beta_true[1] * x1 + u[group_idx] + eps
|
|
171
|
+
|
|
172
|
+
X = np.column_stack([np.ones(n), x1])
|
|
173
|
+
beta, _blup, sigma2, lam_opt = fit_random_intercept(X, y[:, None], group_idx, n_subjects, n_k)
|
|
174
|
+
|
|
175
|
+
np.testing.assert_allclose(beta[:, 0], beta_true, rtol=0, atol=0.25)
|
|
176
|
+
assert 0.5 < sigma2[0] < 2.0
|
|
177
|
+
assert lam_opt[0] > 0.0 # a real random intercept is detected
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def test_random_intercept_matches_statsmodels():
|
|
181
|
+
"""Cross-check REML estimates against statsmodels.MixedLM."""
|
|
182
|
+
sm = pytest.importorskip("statsmodels.api")
|
|
183
|
+
|
|
184
|
+
rng = np.random.default_rng(1)
|
|
185
|
+
n_subjects, n_times = 60, 5
|
|
186
|
+
n = n_subjects * n_times
|
|
187
|
+
group_idx = np.repeat(np.arange(n_subjects), n_times).astype(np.intp)
|
|
188
|
+
n_k = np.bincount(group_idx).astype(np.float64)
|
|
189
|
+
|
|
190
|
+
x1 = rng.standard_normal(n)
|
|
191
|
+
u = rng.normal(0.0, 1.2, n_subjects)
|
|
192
|
+
eps = rng.normal(0.0, 0.9, n)
|
|
193
|
+
y = 1.0 + 0.8 * x1 + u[group_idx] + eps
|
|
194
|
+
|
|
195
|
+
X = np.column_stack([np.ones(n), x1])
|
|
196
|
+
beta, _blup, sigma2, lam_opt = fit_random_intercept(X, y[:, None], group_idx, n_subjects, n_k)
|
|
197
|
+
tau2 = lam_opt[0] * sigma2[0]
|
|
198
|
+
|
|
199
|
+
res = sm.MixedLM(endog=y, exog=X, groups=group_idx).fit(reml=True, method="lbfgs")
|
|
200
|
+
sm_tau2 = float(np.asarray(res.cov_re)[0, 0])
|
|
201
|
+
|
|
202
|
+
np.testing.assert_allclose(beta[:, 0], res.fe_params, rtol=0.02, atol=0.02)
|
|
203
|
+
np.testing.assert_allclose(sigma2[0], res.scale, rtol=0.05)
|
|
204
|
+
np.testing.assert_allclose(tau2, sm_tau2, rtol=0.12, atol=0.05)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|