combatlearn 2.0.0__tar.gz → 2.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. {combatlearn-2.0.0 → combatlearn-2.1.0}/PKG-INFO +13 -6
  2. {combatlearn-2.0.0 → combatlearn-2.1.0}/README.md +12 -6
  3. {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn/__init__.py +1 -1
  4. combatlearn-2.1.0/combatlearn/_mixed.py +165 -0
  5. {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn/core.py +243 -10
  6. {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn/sklearn_api.py +42 -6
  7. {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn.egg-info/PKG-INFO +13 -6
  8. {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn.egg-info/SOURCES.txt +2 -0
  9. {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn.egg-info/requires.txt +1 -0
  10. {combatlearn-2.0.0 → combatlearn-2.1.0}/requirements-dev.txt +1 -0
  11. combatlearn-2.1.0/tests/test_longitudinal.py +204 -0
  12. {combatlearn-2.0.0 → combatlearn-2.1.0}/LICENSE +0 -0
  13. {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn/_utils.py +0 -0
  14. {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn/inspection.py +0 -0
  15. {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn/metrics.py +0 -0
  16. {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn/py.typed +0 -0
  17. {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn/visualization.py +0 -0
  18. {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn.egg-info/dependency_links.txt +0 -0
  19. {combatlearn-2.0.0 → combatlearn-2.1.0}/combatlearn.egg-info/top_level.txt +0 -0
  20. {combatlearn-2.0.0 → combatlearn-2.1.0}/pyproject.toml +0 -0
  21. {combatlearn-2.0.0 → combatlearn-2.1.0}/requirements-docs.txt +0 -0
  22. {combatlearn-2.0.0 → combatlearn-2.1.0}/requirements.txt +0 -0
  23. {combatlearn-2.0.0 → combatlearn-2.1.0}/setup.cfg +0 -0
  24. {combatlearn-2.0.0 → combatlearn-2.1.0}/tests/test_combat.py +0 -0
  25. {combatlearn-2.0.0 → combatlearn-2.1.0}/tests/test_inspection.py +0 -0
  26. {combatlearn-2.0.0 → combatlearn-2.1.0}/tests/test_metrics.py +0 -0
  27. {combatlearn-2.0.0 → combatlearn-2.1.0}/tests/test_visualization.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: combatlearn
3
- Version: 2.0.0
3
+ Version: 2.1.0
4
4
  Summary: Batch-effect harmonization for machine learning frameworks.
5
5
  Author-email: Ettore Rocchi <ettoreroc@gmail.com>
6
6
  License: MIT
@@ -27,6 +27,7 @@ Requires-Dist: ruff>=0.8; extra == "dev"
27
27
  Requires-Dist: pre-commit>=3.0; extra == "dev"
28
28
  Requires-Dist: mypy>=1.0; extra == "dev"
29
29
  Requires-Dist: pandas-stubs; extra == "dev"
30
+ Requires-Dist: statsmodels>=0.14; extra == "dev"
30
31
  Provides-Extra: docs
31
32
  Requires-Dist: sphinx>=7.0.0; extra == "docs"
32
33
  Requires-Dist: pydata-sphinx-theme>=0.15.0; extra == "docs"
@@ -49,10 +50,11 @@ Dynamic: license-file
49
50
 
50
51
  **combatlearn** makes the popular _ComBat_ (and _CovBat_) batch-effect correction algorithm available for use into machine learning frameworks. It lets you harmonise high-dimensional data inside a scikit-learn `Pipeline`, so that cross-validation and grid-search automatically take batch structure into account, **without data leakage**.
51
52
 
52
- **Three methods**:
53
+ **Four methods**:
53
54
  - `method="johnson"` - classic ComBat (Johnson _et al._, 2007)
54
55
  - `method="fortin"` - neuroComBat (Fortin _et al._, 2018)
55
56
  - `method="chen"` - CovBat (Chen _et al._, 2022)
57
+ - `method="longitudinal"` - Longitudinal ComBat for repeated measures (Beer _et al._, 2020)
56
58
 
57
59
  ## Installation
58
60
 
@@ -75,6 +77,7 @@ import pandas as pd
75
77
  from sklearn.pipeline import Pipeline
76
78
  from sklearn.preprocessing import StandardScaler
77
79
  from sklearn.linear_model import LogisticRegression
80
+ from sklearn.model_selection import GridSearchCV
78
81
  from combatlearn import ComBat
79
82
 
80
83
  df = pd.read_csv("data.csv", index_col=0)
@@ -125,18 +128,20 @@ The following section provides a detailed explanation of all parameters availabl
125
128
  | Parameter | Type | Default | Description |
126
129
  | --- | --- | --- | --- |
127
130
  | `batch` | array-like or pd.Series | **required** | Vector indicating batch assignment for each sample. This is used to estimate and remove batch effects. |
128
- | `discrete_covariates` | array-like, pd.Series, or pd.DataFrame | `None` | Optional categorical covariates (e.g., sex, site). Only used in `"fortin"` and `"chen"` methods. |
129
- | `continuous_covariates` | array-like, pd.Series or pd.DataFrame | `None` | Optional continuous covariates (e.g., age). Only used in `"fortin"` and `"chen"` methods. |
131
+ | `discrete_covariates` | array-like, pd.Series, or pd.DataFrame | `None` | Optional categorical covariates (e.g., sex, site). Only used in `"fortin"`, `"chen"`, and `"longitudinal"` methods. |
132
+ | `continuous_covariates` | array-like, pd.Series or pd.DataFrame | `None` | Optional continuous covariates (e.g., age). Only used in `"fortin"`, `"chen"`, and `"longitudinal"` methods. |
133
+ | `subject_id` | array-like or pd.Series | `None` | Subject/individual labels for the random intercept. **Required** for `method="longitudinal"`, ignored otherwise. |
134
+ | `time_covariate` | array-like or pd.Series | `None` | Optional continuous time variable for repeated measures. Only used in `"longitudinal"`. |
130
135
 
131
136
  ### Algorithm Options
132
137
 
133
138
  | Parameter | Type | Default | Description |
134
139
  | --- | --- | --- | --- |
135
- | `method` | str | `"johnson"` | ComBat method to use: <ul><li>`"johnson"` - Classical ComBat (_Johnson et al. 2007_)</li><li>`"fortin"` - ComBat with covariates (_Fortin et al. 2018_)</li><li>`"chen"` - CovBat, PCA-based correction (_Chen et al. 2022_)</li></ul> |
140
+ | `method` | str | `"johnson"` | ComBat method to use: <ul><li>`"johnson"` - Classical ComBat (_Johnson et al. 2007_)</li><li>`"fortin"` - ComBat with covariates (_Fortin et al. 2018_)</li><li>`"chen"` - CovBat, PCA-based correction (_Chen et al. 2022_)</li><li>`"longitudinal"` - Longitudinal ComBat with a per-subject random intercept (_Beer et al. 2020_)</li></ul> |
136
141
  | `parametric` | bool | `True` | Whether to use the **parametric empirical Bayes** formulation. If `False`, a non-parametric iterative scheme is used. |
137
142
  | `mean_only` | bool | `False` | If `True`, only the **mean** is corrected, while variances are left unchanged. Useful for preserving variance structure in the data. |
138
143
  | `reference_batch` | str or `None` | `None` | If specified, acts as a reference batch - other batches will be corrected to match this one. |
139
- | `covbat_cov_thresh` | float, int | `0.9` | For `"chen"` method only: Cumulative variance threshold $]0,1[$ to retain PCs in PCA space (e.g., 0.9 = retain 90% explained variance). If an integer is provided, it represents the number of principal components to use. |
144
+ | `covbat_cov_thresh` | float, int | `0.9` | For `"chen"` method only: Cumulative variance threshold in `(0, 1]` to retain PCs in PCA space (e.g., 0.9 = retain 90% explained variance). If an integer is provided, it represents the number of principal components to use. |
140
145
  | `eps` | float | `1e-8` | Small jitter value added to variances to prevent divide-by-zero errors during standardization. |
141
146
 
142
147
 
@@ -190,3 +195,5 @@ Please consider citing the original papers:
190
195
  - [**neuroCombat**](https://github.com/Jfortin1/neuroCombat) - Fortin JP et al. _Neuroimage_. 2018. doi: [10.1016/j.neuroimage.2017.11.024](https://doi.org/10.1016/j.neuroimage.2017.11.024)
191
196
 
192
197
  - [**CovBat**](https://github.com/andy1764/CovBat_Harmonization) - Chen AA et al. _Hum Brain Mapp_. 2022. doi: [10.1002/hbm.25688](https://doi.org/10.1002/hbm.25688)
198
+
199
+ - [**Longitudinal ComBat**](https://github.com/jcbeer/longCombat) - Beer JC et al. _Neuroimage_. 2020. doi: [10.1016/j.neuroimage.2020.117129](https://doi.org/10.1016/j.neuroimage.2020.117129)
@@ -12,10 +12,11 @@
12
12
 
13
13
  **combatlearn** makes the popular _ComBat_ (and _CovBat_) batch-effect correction algorithm available for use into machine learning frameworks. It lets you harmonise high-dimensional data inside a scikit-learn `Pipeline`, so that cross-validation and grid-search automatically take batch structure into account, **without data leakage**.
14
14
 
15
- **Three methods**:
15
+ **Four methods**:
16
16
  - `method="johnson"` - classic ComBat (Johnson _et al._, 2007)
17
17
  - `method="fortin"` - neuroComBat (Fortin _et al._, 2018)
18
18
  - `method="chen"` - CovBat (Chen _et al._, 2022)
19
+ - `method="longitudinal"` - Longitudinal ComBat for repeated measures (Beer _et al._, 2020)
19
20
 
20
21
  ## Installation
21
22
 
@@ -38,6 +39,7 @@ import pandas as pd
38
39
  from sklearn.pipeline import Pipeline
39
40
  from sklearn.preprocessing import StandardScaler
40
41
  from sklearn.linear_model import LogisticRegression
42
+ from sklearn.model_selection import GridSearchCV
41
43
  from combatlearn import ComBat
42
44
 
43
45
  df = pd.read_csv("data.csv", index_col=0)
@@ -88,18 +90,20 @@ The following section provides a detailed explanation of all parameters availabl
88
90
  | Parameter | Type | Default | Description |
89
91
  | --- | --- | --- | --- |
90
92
  | `batch` | array-like or pd.Series | **required** | Vector indicating batch assignment for each sample. This is used to estimate and remove batch effects. |
91
- | `discrete_covariates` | array-like, pd.Series, or pd.DataFrame | `None` | Optional categorical covariates (e.g., sex, site). Only used in `"fortin"` and `"chen"` methods. |
92
- | `continuous_covariates` | array-like, pd.Series or pd.DataFrame | `None` | Optional continuous covariates (e.g., age). Only used in `"fortin"` and `"chen"` methods. |
93
+ | `discrete_covariates` | array-like, pd.Series, or pd.DataFrame | `None` | Optional categorical covariates (e.g., sex, site). Only used in `"fortin"`, `"chen"`, and `"longitudinal"` methods. |
94
+ | `continuous_covariates` | array-like, pd.Series or pd.DataFrame | `None` | Optional continuous covariates (e.g., age). Only used in `"fortin"`, `"chen"`, and `"longitudinal"` methods. |
95
+ | `subject_id` | array-like or pd.Series | `None` | Subject/individual labels for the random intercept. **Required** for `method="longitudinal"`, ignored otherwise. |
96
+ | `time_covariate` | array-like or pd.Series | `None` | Optional continuous time variable for repeated measures. Only used in `"longitudinal"`. |
93
97
 
94
98
  ### Algorithm Options
95
99
 
96
100
  | Parameter | Type | Default | Description |
97
101
  | --- | --- | --- | --- |
98
- | `method` | str | `"johnson"` | ComBat method to use: <ul><li>`"johnson"` - Classical ComBat (_Johnson et al. 2007_)</li><li>`"fortin"` - ComBat with covariates (_Fortin et al. 2018_)</li><li>`"chen"` - CovBat, PCA-based correction (_Chen et al. 2022_)</li></ul> |
102
+ | `method` | str | `"johnson"` | ComBat method to use: <ul><li>`"johnson"` - Classical ComBat (_Johnson et al. 2007_)</li><li>`"fortin"` - ComBat with covariates (_Fortin et al. 2018_)</li><li>`"chen"` - CovBat, PCA-based correction (_Chen et al. 2022_)</li><li>`"longitudinal"` - Longitudinal ComBat with a per-subject random intercept (_Beer et al. 2020_)</li></ul> |
99
103
  | `parametric` | bool | `True` | Whether to use the **parametric empirical Bayes** formulation. If `False`, a non-parametric iterative scheme is used. |
100
104
  | `mean_only` | bool | `False` | If `True`, only the **mean** is corrected, while variances are left unchanged. Useful for preserving variance structure in the data. |
101
105
  | `reference_batch` | str or `None` | `None` | If specified, acts as a reference batch - other batches will be corrected to match this one. |
102
- | `covbat_cov_thresh` | float, int | `0.9` | For `"chen"` method only: Cumulative variance threshold $]0,1[$ to retain PCs in PCA space (e.g., 0.9 = retain 90% explained variance). If an integer is provided, it represents the number of principal components to use. |
106
+ | `covbat_cov_thresh` | float, int | `0.9` | For `"chen"` method only: Cumulative variance threshold in `(0, 1]` to retain PCs in PCA space (e.g., 0.9 = retain 90% explained variance). If an integer is provided, it represents the number of principal components to use. |
103
107
  | `eps` | float | `1e-8` | Small jitter value added to variances to prevent divide-by-zero errors during standardization. |
104
108
 
105
109
 
@@ -152,4 +156,6 @@ Please consider citing the original papers:
152
156
 
153
157
  - [**neuroCombat**](https://github.com/Jfortin1/neuroCombat) - Fortin JP et al. _Neuroimage_. 2018. doi: [10.1016/j.neuroimage.2017.11.024](https://doi.org/10.1016/j.neuroimage.2017.11.024)
154
158
 
155
- - [**CovBat**](https://github.com/andy1764/CovBat_Harmonization) - Chen AA et al. _Hum Brain Mapp_. 2022. doi: [10.1002/hbm.25688](https://doi.org/10.1002/hbm.25688)
159
+ - [**CovBat**](https://github.com/andy1764/CovBat_Harmonization) - Chen AA et al. _Hum Brain Mapp_. 2022. doi: [10.1002/hbm.25688](https://doi.org/10.1002/hbm.25688)
160
+
161
+ - [**Longitudinal ComBat**](https://github.com/jcbeer/longCombat) - Beer JC et al. _Neuroimage_. 2020. doi: [10.1016/j.neuroimage.2020.117129](https://doi.org/10.1016/j.neuroimage.2020.117129)
@@ -1,5 +1,5 @@
1
1
  from .sklearn_api import ComBat
2
2
 
3
3
  __all__ = ["ComBat"]
4
- __version__ = "2.0.0"
4
+ __version__ = "2.1.0"
5
5
  __author__ = "Ettore Rocchi"
@@ -0,0 +1,165 @@
1
+ """Random-intercept linear mixed model fitting for Longitudinal ComBat.
2
+
3
+ Fits, for every feature (column of ``Y``) simultaneously, the model
4
+
5
+ y = X @ beta + Z @ u + eps, u ~ N(0, tau^2 I), eps ~ N(0, sigma^2 I)
6
+
7
+ where ``Z`` is the subject (random-intercept) indicator matrix. Estimation is
8
+ by REML, profiling out ``beta`` and ``sigma^2`` so that only the scalar
9
+ variance ratio ``lambda = tau^2 / sigma^2`` is optimized per feature.
10
+
11
+ The single random-intercept grouping makes the marginal covariance
12
+ ``V = sigma^2 (I + lambda Z Z^T)`` block diagonal (one block per subject), so
13
+ ``V^{-1}`` is available in closed form via Sherman-Morrison and every operation
14
+ is O(n) per feature. The variance ratio is found by a vectorized grid search
15
+ over ``lambda`` followed by a parabolic refinement, after which the fixed
16
+ effects, residual variance, and random-intercept BLUPs are computed in a single
17
+ per-feature pass.
18
+
19
+ This keeps Longitudinal ComBat dependency-free (numpy only); correctness is
20
+ cross-checked against ``statsmodels.MixedLM`` in the test suite.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import numpy as np
26
+ import numpy.typing as npt
27
+
28
+ FloatArray = npt.NDArray[np.float64]
29
+ IntArray = npt.NDArray[np.intp]
30
+
31
+
32
+ def _group_sums(M: FloatArray, group_idx: IntArray, n_groups: int) -> FloatArray:
33
+ """Sum the rows of ``M`` (n, c) within each group, returning (n_groups, c)."""
34
+ out = np.zeros((n_groups, M.shape[1]), dtype=np.float64)
35
+ np.add.at(out, group_idx, M)
36
+ return out
37
+
38
+
39
+ def _w_apply(M: FloatArray, factor: FloatArray, group_idx: IntArray, n_groups: int) -> FloatArray:
40
+ """Apply ``W = (I + lambda Z Z^T)^{-1}`` to the columns of ``M``.
41
+
42
+ ``factor[k] = lambda / (1 + lambda * n_k)`` is the Sherman-Morrison
43
+ per-group correction (``n_k`` = number of observations in group ``k``).
44
+ """
45
+ gs = _group_sums(M, group_idx, n_groups)
46
+ return M - factor[group_idx][:, None] * gs[group_idx]
47
+
48
+
49
+ def _reml_objective(
50
+ lam: float,
51
+ X: FloatArray,
52
+ Y: FloatArray,
53
+ group_idx: IntArray,
54
+ n_groups: int,
55
+ n_k: FloatArray,
56
+ n: int,
57
+ p: int,
58
+ ridge: float,
59
+ ) -> FloatArray:
60
+ """Profiled REML objective (to minimize) evaluated for all features at ``lam``."""
61
+ factor = lam / (1.0 + lam * n_k)
62
+ WX = _w_apply(X, factor, group_idx, n_groups)
63
+ A = X.T @ WX
64
+ A.flat[:: p + 1] += ridge
65
+ beta = np.linalg.solve(A, X.T @ _w_apply(Y, factor, group_idx, n_groups))
66
+ R = Y - X @ beta
67
+ WR = _w_apply(R, factor, group_idx, n_groups)
68
+ rss = np.maximum(np.einsum("ng,ng->g", R, WR), 1e-12)
69
+ logdet_v = float(np.sum(np.log1p(lam * n_k)))
70
+ _, logdet_a = np.linalg.slogdet(A)
71
+ result = logdet_v + (n - p) * np.log(rss) + float(logdet_a)
72
+ return np.asarray(result, dtype=np.float64)
73
+
74
+
75
+ def fit_random_intercept(
76
+ X: FloatArray,
77
+ Y: FloatArray,
78
+ group_idx: IntArray,
79
+ n_groups: int,
80
+ n_k: FloatArray,
81
+ *,
82
+ eps: float = 1e-8,
83
+ n_grid: int = 80,
84
+ ) -> tuple[FloatArray, FloatArray, FloatArray, FloatArray]:
85
+ """Fit a random-intercept LME per feature by REML.
86
+
87
+ Parameters
88
+ ----------
89
+ X : ndarray of shape (n, p)
90
+ Fixed-effects design matrix (shared across features).
91
+ Y : ndarray of shape (n, n_features)
92
+ Response matrix; one column per feature.
93
+ group_idx : ndarray of shape (n,)
94
+ Integer group (subject) code in ``[0, n_groups)`` for each observation.
95
+ n_groups : int
96
+ Number of distinct subjects.
97
+ n_k : ndarray of shape (n_groups,)
98
+ Number of observations per subject.
99
+ eps : float, default=1e-8
100
+ Numerical jitter; also scales the diagonal ridge on ``X^T W X``.
101
+ n_grid : int, default=80
102
+ Number of log-spaced ``lambda`` grid points for the variance-ratio search.
103
+
104
+ Returns
105
+ -------
106
+ beta : ndarray of shape (p, n_features)
107
+ Estimated fixed-effects coefficients.
108
+ blup : ndarray of shape (n_groups, n_features)
109
+ Best linear unbiased predictors of the subject random intercepts.
110
+ sigma2 : ndarray of shape (n_features,)
111
+ Estimated residual variance per feature.
112
+ lam_opt : ndarray of shape (n_features,)
113
+ Estimated variance ratio ``tau^2 / sigma^2`` per feature.
114
+ """
115
+ X = np.asarray(X, dtype=np.float64)
116
+ Y = np.asarray(Y, dtype=np.float64)
117
+ group_idx = np.asarray(group_idx, dtype=np.intp)
118
+ n_k = np.asarray(n_k, dtype=np.float64)
119
+ n, p = X.shape
120
+ n_features = Y.shape[1]
121
+ ridge = eps * float(np.trace(X.T @ X)) / max(p, 1)
122
+
123
+ # Vectorized grid search for the variance ratio (per feature).
124
+ log_grid = np.linspace(-4.0, 4.0, n_grid)
125
+ lam_grid = np.concatenate([[0.0], 10.0**log_grid])
126
+ fvals = np.empty((lam_grid.size, n_features), dtype=np.float64)
127
+ for i, lam in enumerate(lam_grid):
128
+ fvals[i] = _reml_objective(lam, X, Y, group_idx, n_groups, n_k, n, p, ridge)
129
+ best = np.argmin(fvals, axis=0)
130
+ lam_opt = lam_grid[best].astype(np.float64)
131
+
132
+ # Vectorized parabolic refinement (log10 space) for interior positive optima.
133
+ interior = (best >= 2) & (best <= n_grid - 1)
134
+ if np.any(interior):
135
+ bi = best[interior]
136
+ f_lo = fvals[bi - 1, interior]
137
+ f0 = fvals[bi, interior]
138
+ f_hi = fvals[bi + 1, interior]
139
+ denom = f_lo - 2.0 * f0 + f_hi
140
+ delta = np.where(np.abs(denom) > 1e-12, 0.5 * (f_lo - f_hi) / denom, 0.0)
141
+ delta = np.clip(delta, -1.0, 1.0)
142
+ step = log_grid[1] - log_grid[0]
143
+ lam_opt[interior] = 10.0 ** (log_grid[bi - 1] + delta * step)
144
+
145
+ # Final per-feature fixed effects, residual variance, and random-intercept BLUPs.
146
+ beta = np.empty((p, n_features), dtype=np.float64)
147
+ sigma2 = np.empty(n_features, dtype=np.float64)
148
+ blup = np.zeros((n_groups, n_features), dtype=np.float64)
149
+ denom_sigma = max(n - p, 1)
150
+ for g in range(n_features):
151
+ lam = float(lam_opt[g])
152
+ factor = lam / (1.0 + lam * n_k)
153
+ A = X.T @ _w_apply(X, factor, group_idx, n_groups)
154
+ A.flat[:: p + 1] += ridge
155
+ y = Y[:, g : g + 1]
156
+ b = np.linalg.solve(A, X.T @ _w_apply(y, factor, group_idx, n_groups))[:, 0]
157
+ beta[:, g] = b
158
+ r = Y[:, g] - X @ b
159
+ Wr = _w_apply(r[:, None], factor, group_idx, n_groups)[:, 0]
160
+ sigma2[g] = float(r @ Wr) / denom_sigma
161
+ s_k = np.zeros(n_groups, dtype=np.float64)
162
+ np.add.at(s_k, group_idx, r)
163
+ blup[:, g] = lam * s_k / (1.0 + lam * n_k)
164
+
165
+ return beta, blup, sigma2, lam_opt
@@ -1,9 +1,10 @@
1
1
  """ComBat algorithm core.
2
2
 
3
- `ComBatModel` implements three variants of the ComBat algorithm:
3
+ `ComBatModel` implements four variants of the ComBat algorithm:
4
4
  * Johnson et al. (2007) vanilla ComBat (method="johnson")
5
5
  * Fortin et al. (2018) extension with covariates (method="fortin")
6
6
  * Chen et al. (2022) CovBat (method="chen")
7
+ * Beer et al. (2020) Longitudinal ComBat (method="longitudinal")
7
8
  """
8
9
 
9
10
  from __future__ import annotations
@@ -17,6 +18,8 @@ import numpy.typing as npt
17
18
  import pandas as pd
18
19
  from sklearn.decomposition import PCA
19
20
 
21
+ from ._mixed import fit_random_intercept
22
+
20
23
  ArrayLike: TypeAlias = pd.DataFrame | pd.Series | npt.NDArray[Any]
21
24
  FloatArray: TypeAlias = npt.NDArray[np.float64]
22
25
 
@@ -26,10 +29,12 @@ class ComBatModel:
26
29
 
27
30
  Parameters
28
31
  ----------
29
- method : {'johnson', 'fortin', 'chen'}, default='johnson'
32
+ method : {'johnson', 'fortin', 'chen', 'longitudinal'}, default='johnson'
30
33
  * 'johnson' - classic ComBat.
31
34
  * 'fortin' - covariate-aware ComBat.
32
35
  * 'chen' - CovBat, PCA-based ComBat.
36
+ * 'longitudinal' - Longitudinal ComBat; requires a per-sample
37
+ ``subject_id`` for the random intercept.
33
38
  parametric : bool, default=True
34
39
  Use the parametric empirical Bayes variant.
35
40
  mean_only : bool, default=False
@@ -47,7 +52,7 @@ class ComBatModel:
47
52
  def __init__(
48
53
  self,
49
54
  *,
50
- method: Literal["johnson", "fortin", "chen"] = "johnson",
55
+ method: Literal["johnson", "fortin", "chen", "longitudinal"] = "johnson",
51
56
  parametric: bool = True,
52
57
  mean_only: bool = False,
53
58
  reference_batch: str | None = None,
@@ -77,6 +82,8 @@ class ComBatModel:
77
82
  self._batch_levels_pc: pd.Index
78
83
  self._pc_gamma_star: FloatArray
79
84
  self._pc_delta_star: FloatArray
85
+ self._re_subject_levels: pd.Index
86
+ self._re_blup: FloatArray
80
87
 
81
88
  # Validate covbat_cov_thresh
82
89
  if isinstance(self.covbat_cov_thresh, float):
@@ -138,6 +145,8 @@ class ComBatModel:
138
145
  batch: ArrayLike,
139
146
  discrete_covariates: ArrayLike | None = None,
140
147
  continuous_covariates: ArrayLike | None = None,
148
+ subject_id: ArrayLike | None = None,
149
+ time_covariate: ArrayLike | None = None,
141
150
  ) -> ComBatModel:
142
151
  """Fit the ComBat model.
143
152
 
@@ -150,9 +159,15 @@ class ComBatModel:
150
159
  batch : array-like of shape (n_samples,)
151
160
  Batch labels for each sample.
152
161
  discrete_covariates : array-like or None, default=None
153
- Categorical covariates to protect (Fortin/Chen only).
162
+ Categorical covariates to protect (Fortin/Chen/Longitudinal only).
154
163
  continuous_covariates : array-like or None, default=None
155
- Continuous covariates to protect (Fortin/Chen only).
164
+ Continuous covariates to protect (Fortin/Chen/Longitudinal only).
165
+ subject_id : array-like or None, default=None
166
+ Subject/individual labels for the random intercept. Required for
167
+ ``method='longitudinal'``, ignored otherwise.
168
+ time_covariate : array-like or None, default=None
169
+ Continuous time variable for repeated measures (Longitudinal only).
170
+ Added to the fixed-effects design alongside ``continuous_covariates``.
156
171
 
157
172
  Returns
158
173
  -------
@@ -160,10 +175,10 @@ class ComBatModel:
160
175
  Fitted model.
161
176
  """
162
177
  method = self.method.lower()
163
- if method not in {"johnson", "fortin", "chen"}:
178
+ if method not in {"johnson", "fortin", "chen", "longitudinal"}:
164
179
  raise ValueError(
165
180
  f"method={self.method!r} is not recognized. "
166
- f"Expected one of 'johnson', 'fortin', or 'chen'."
181
+ f"Expected one of 'johnson', 'fortin', 'chen', or 'longitudinal'."
167
182
  )
168
183
  if not isinstance(X, pd.DataFrame):
169
184
  X = pd.DataFrame(X)
@@ -180,6 +195,13 @@ class ComBatModel:
180
195
  f"Check for typos or trailing whitespace in batch labels."
181
196
  )
182
197
 
198
+ if method != "longitudinal" and (subject_id is not None or time_covariate is not None):
199
+ warnings.warn(
200
+ "`subject_id` and `time_covariate` are only used when "
201
+ "method='longitudinal'; they are ignored here.",
202
+ stacklevel=2,
203
+ )
204
+
183
205
  if method == "johnson":
184
206
  if disc is not None or cont is not None:
185
207
  warnings.warn("Covariates are ignored when using method='johnson'.", stacklevel=2)
@@ -188,6 +210,8 @@ class ComBatModel:
188
210
  self._fit_fortin(X, batch, disc, cont)
189
211
  elif method == "chen":
190
212
  self._fit_chen(X, batch, disc, cont)
213
+ elif method == "longitudinal":
214
+ self._fit_longitudinal(X, batch, disc, cont, subject_id, time_covariate)
191
215
  return self
192
216
 
193
217
  def _fit_johnson(self, X: pd.DataFrame, batch: pd.Series) -> None:
@@ -431,6 +455,153 @@ class ComBatModel:
431
455
  self._pc_gamma_star = gamma_star
432
456
  self._pc_delta_star = delta_star
433
457
 
458
+ def _build_longitudinal_design(
459
+ self,
460
+ X: pd.DataFrame,
461
+ batch: pd.Series,
462
+ disc: pd.DataFrame | None,
463
+ cont: pd.DataFrame | None,
464
+ time: pd.DataFrame | None,
465
+ ) -> tuple[FloatArray, list[str]]:
466
+ """Assemble the fixed-effects design [batch dummies | covariates | time].
467
+
468
+ Returns the design matrix and the list of non-batch column names.
469
+ """
470
+ batch_dummies = pd.get_dummies(batch, drop_first=False).astype(float)
471
+ batch_dummies = batch_dummies.reindex(columns=self._batch_levels, fill_value=0.0)
472
+
473
+ nonbatch_parts: list[pd.DataFrame] = []
474
+ if disc is not None:
475
+ nonbatch_parts.append(
476
+ pd.get_dummies(disc.astype("category"), drop_first=True).astype(float)
477
+ )
478
+ if cont is not None:
479
+ nonbatch_parts.append(cont.astype(float))
480
+ if time is not None:
481
+ nonbatch_parts.append(time.astype(float))
482
+
483
+ if nonbatch_parts:
484
+ nonbatch_df = pd.concat(nonbatch_parts, axis=1)
485
+ else:
486
+ nonbatch_df = pd.DataFrame(index=batch_dummies.index)
487
+
488
+ design_df = pd.concat([batch_dummies, nonbatch_df], axis=1)
489
+ n_batch = len(self._batch_levels)
490
+ nonbatch_columns = design_df.columns[n_batch:].tolist()
491
+ return design_df.values.astype(np.float64), nonbatch_columns
492
+
493
+ def _fit_longitudinal(
494
+ self,
495
+ X: pd.DataFrame,
496
+ batch: pd.Series,
497
+ disc: pd.DataFrame | None,
498
+ cont: pd.DataFrame | None,
499
+ subject_id: ArrayLike | None,
500
+ time_covariate: ArrayLike | None,
501
+ ) -> None:
502
+ """Longitudinal ComBat (*Beer et al.* 2020).
503
+
504
+ Identical to Fortin's standardize -> per-batch gamma/delta -> empirical
505
+ Bayes -> remove pipeline, except the fixed-effects mean model is fit with
506
+ a per-subject random intercept (REML), and the random-intercept BLUP is
507
+ included in the standardization mean.
508
+ """
509
+ if subject_id is None:
510
+ raise ValueError(
511
+ "method='longitudinal' requires `subject_id` (one label per sample "
512
+ "identifying the subject/individual used for the random intercept)."
513
+ )
514
+
515
+ self._batch_levels = batch.cat.categories
516
+ n_batch = len(self._batch_levels)
517
+ n_samples = len(X)
518
+
519
+ counts = batch.value_counts()
520
+ small = counts[counts < 2]
521
+ if len(small):
522
+ lvl = small.index[0]
523
+ raise ValueError(
524
+ f"Batch '{lvl}' has only {int(small.iloc[0])} sample(s). ComBat requires "
525
+ f">= 2 samples per batch. Consider merging small batches or removing them."
526
+ )
527
+
528
+ time = self._to_df(time_covariate, X.index, "time_covariate")
529
+ design, self._nonbatch_columns = self._build_longitudinal_design(X, batch, disc, cont, time)
530
+ self._design_cond = float(la.cond(design)) if design.shape[1] else 1.0
531
+ self._n_batch = n_batch
532
+ self._p_design = design.shape[1]
533
+
534
+ subj = self._as_series(subject_id, X.index, "subject_id")
535
+ self._re_subject_levels = subj.cat.categories
536
+ group_idx = subj.cat.codes.to_numpy().astype(np.intp)
537
+ n_groups = len(self._re_subject_levels)
538
+ n_k = np.bincount(group_idx, minlength=n_groups).astype(np.float64)
539
+
540
+ beta, blup, sigma2, _ = fit_random_intercept(
541
+ design, X.values.astype(np.float64), group_idx, n_groups, n_k, eps=self.eps
542
+ )
543
+ self._re_blup = blup
544
+
545
+ beta_batch = beta[:n_batch]
546
+ self._beta_hat_nonbatch = beta[n_batch:]
547
+
548
+ n_per_batch_arr = np.asarray(
549
+ batch.value_counts().reindex(self._batch_levels).astype(int).values, dtype=np.float64
550
+ )
551
+ self._n_per_batch = dict(zip(self._batch_levels, n_per_batch_arr.astype(int), strict=True))
552
+
553
+ grand_mean = (n_per_batch_arr / n_samples) @ beta_batch
554
+ self._grand_mean = pd.Series(grand_mean, index=X.columns)
555
+ var_pooled = sigma2 + self.eps
556
+ self._pooled_var = pd.Series(var_pooled, index=X.columns)
557
+
558
+ blup_per_row = blup[group_idx]
559
+ stand_mean = grand_mean + design[:, n_batch:] @ self._beta_hat_nonbatch + blup_per_row
560
+ Xs = (X.values - stand_mean) / np.sqrt(var_pooled)
561
+
562
+ gamma_hat = np.vstack(
563
+ [Xs[(batch == lvl).to_numpy()].mean(axis=0) for lvl in self._batch_levels]
564
+ )
565
+ delta_hat = np.vstack(
566
+ [
567
+ Xs[(batch == lvl).to_numpy()].var(axis=0, ddof=1) + self.eps
568
+ for lvl in self._batch_levels
569
+ ]
570
+ )
571
+
572
+ b_names = [str(lvl) for lvl in self._batch_levels]
573
+ if self.mean_only:
574
+ gamma_star = self._shrink_gamma(
575
+ gamma_hat,
576
+ delta_hat,
577
+ n_per_batch_arr,
578
+ parametric=self.parametric,
579
+ batch_names=b_names,
580
+ )
581
+ delta_star = np.ones_like(delta_hat)
582
+ else:
583
+ gamma_star, delta_star = self._shrink_gamma_delta(
584
+ gamma_hat,
585
+ delta_hat,
586
+ n_per_batch_arr,
587
+ parametric=self.parametric,
588
+ batch_names=b_names,
589
+ )
590
+
591
+ if self.reference_batch is not None:
592
+ ref_idx = list(self._batch_levels).index(self.reference_batch)
593
+ gamma_ref = gamma_star[ref_idx]
594
+ delta_ref = delta_star[ref_idx]
595
+ gamma_star = gamma_star - gamma_ref
596
+ if not self.mean_only:
597
+ delta_star = delta_star / delta_ref
598
+ self._reference_batch_idx = ref_idx
599
+ else:
600
+ self._reference_batch_idx = None
601
+
602
+ self._gamma_star = gamma_star
603
+ self._delta_star = delta_star
604
+
434
605
  def _shrink_gamma_delta(
435
606
  self,
436
607
  gamma_hat: FloatArray,
@@ -620,6 +791,8 @@ class ComBatModel:
620
791
  batch: ArrayLike,
621
792
  discrete_covariates: ArrayLike | None = None,
622
793
  continuous_covariates: ArrayLike | None = None,
794
+ subject_id: ArrayLike | None = None,
795
+ time_covariate: ArrayLike | None = None,
623
796
  ) -> pd.DataFrame:
624
797
  """Transform the data using fitted ComBat parameters.
625
798
 
@@ -630,9 +803,14 @@ class ComBatModel:
630
803
  batch : array-like of shape (n_samples,)
631
804
  Batch labels for each sample.
632
805
  discrete_covariates : array-like or None, default=None
633
- Categorical covariates (Fortin/Chen only).
806
+ Categorical covariates (Fortin/Chen/Longitudinal only).
634
807
  continuous_covariates : array-like or None, default=None
635
- Continuous covariates (Fortin/Chen only).
808
+ Continuous covariates (Fortin/Chen/Longitudinal only).
809
+ subject_id : array-like or None, default=None
810
+ Subject labels for the random intercept (Longitudinal only). Unseen
811
+ subjects fall back to a zero random intercept.
812
+ time_covariate : array-like or None, default=None
813
+ Continuous time variable (Longitudinal only).
636
814
 
637
815
  Returns
638
816
  -------
@@ -665,9 +843,12 @@ class ComBatModel:
665
843
  return self._transform_fortin(X, batch, disc, cont)
666
844
  elif method == "chen":
667
845
  return self._transform_chen(X, batch, disc, cont)
846
+ elif method == "longitudinal":
847
+ return self._transform_longitudinal(X, batch, disc, cont, subject_id, time_covariate)
668
848
  else:
669
849
  raise ValueError(
670
- f"Unknown method={method!r}. Expected one of 'johnson', 'fortin', or 'chen'."
850
+ f"Unknown method={method!r}. Expected one of 'johnson', 'fortin', "
851
+ f"'chen', or 'longitudinal'."
671
852
  )
672
853
 
673
854
  def _transform_johnson(self, X: pd.DataFrame, batch: pd.Series) -> pd.DataFrame:
@@ -773,3 +954,55 @@ class ComBatModel:
773
954
 
774
955
  X_recon = self._covbat_pca.inverse_transform(scores_adj)
775
956
  return pd.DataFrame(X_recon, index=X.index, columns=X.columns)
957
+
958
+ def _transform_longitudinal(
959
+ self,
960
+ X: pd.DataFrame,
961
+ batch: pd.Series,
962
+ disc: pd.DataFrame | None,
963
+ cont: pd.DataFrame | None,
964
+ subject_id: ArrayLike | None,
965
+ time_covariate: ArrayLike | None,
966
+ ) -> pd.DataFrame:
967
+ """Longitudinal transform implementation."""
968
+ if subject_id is None:
969
+ raise ValueError("method='longitudinal' requires `subject_id` at transform time.")
970
+
971
+ n_features = len(self._grand_mean)
972
+ time = self._to_df(time_covariate, X.index, "time_covariate")
973
+ design, nonbatch_columns = self._build_longitudinal_design(X, batch, disc, cont, time)
974
+ # Realign non-batch columns to those seen at fit (handles unseen covariate levels).
975
+ nonbatch_design = pd.DataFrame(
976
+ design[:, self._n_batch :],
977
+ index=X.index,
978
+ columns=nonbatch_columns,
979
+ ).reindex(columns=self._nonbatch_columns, fill_value=0.0)
980
+
981
+ subj = self._as_series(subject_id, X.index, "subject_id")
982
+ level_pos = {lvl: i for i, lvl in enumerate(self._re_subject_levels)}
983
+ codes = np.array([level_pos.get(s, -1) for s in subj.to_numpy()], dtype=np.intp)
984
+ blup_per_row = np.zeros((len(X.index), n_features), dtype=np.float64)
985
+ seen = codes >= 0
986
+ if seen.any():
987
+ blup_per_row[seen] = self._re_blup[codes[seen]]
988
+
989
+ grand_vals = np.asarray(self._grand_mean.values, dtype=np.float64)
990
+ pooled_vals = np.asarray(self._pooled_var.values, dtype=np.float64)
991
+ stand_mean = grand_vals + nonbatch_design.values @ self._beta_hat_nonbatch + blup_per_row
992
+ Xs = (X.values - stand_mean) / np.sqrt(pooled_vals)
993
+
994
+ for i, lvl in enumerate(self._batch_levels):
995
+ idx = (batch == lvl).to_numpy()
996
+ if not idx.any():
997
+ continue
998
+ if self.reference_batch is not None and lvl == self.reference_batch:
999
+ continue
1000
+ g = self._gamma_star[i]
1001
+ d = self._delta_star[i]
1002
+ if self.mean_only:
1003
+ Xs[idx] = Xs[idx] - g
1004
+ else:
1005
+ Xs[idx] = (Xs[idx] - g) / np.sqrt(d)
1006
+
1007
+ X_adj = Xs * np.sqrt(pooled_vals) + stand_mean
1008
+ return pd.DataFrame(X_adj, index=X.index, columns=X.columns, dtype=float)
@@ -27,10 +27,15 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
27
27
  batch : array-like of shape (n_samples,)
28
28
  Batch labels for each sample.
29
29
  discrete_covariates : array-like, optional
30
- Categorical covariates to protect (Fortin/Chen only).
30
+ Categorical covariates to protect (Fortin/Chen/Longitudinal only).
31
31
  continuous_covariates : array-like, optional
32
- Continuous covariates to protect (Fortin/Chen only).
33
- method : {'johnson', 'fortin', 'chen'}, default='johnson'
32
+ Continuous covariates to protect (Fortin/Chen/Longitudinal only).
33
+ subject_id : array-like, optional
34
+ Subject/individual labels for the random intercept. Required for
35
+ ``method='longitudinal'``, ignored otherwise.
36
+ time_covariate : array-like, optional
37
+ Continuous time variable for repeated measures (Longitudinal only).
38
+ method : {'johnson', 'fortin', 'chen', 'longitudinal'}, default='johnson'
34
39
  ComBat variant to use.
35
40
  parametric : bool, default=True
36
41
  Use parametric empirical Bayes.
@@ -53,6 +58,8 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
53
58
  *,
54
59
  discrete_covariates: ArrayLike | None = None,
55
60
  continuous_covariates: ArrayLike | None = None,
61
+ subject_id: ArrayLike | None = None,
62
+ time_covariate: ArrayLike | None = None,
56
63
  method: str = "johnson",
57
64
  parametric: bool = True,
58
65
  mean_only: bool = False,
@@ -63,6 +70,8 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
63
70
  self.batch = batch
64
71
  self.discrete_covariates = discrete_covariates
65
72
  self.continuous_covariates = continuous_covariates
73
+ self.subject_id = subject_id
74
+ self.time_covariate = time_covariate
66
75
  self.method = method
67
76
  self.parametric = parametric
68
77
  self.mean_only = mean_only
@@ -85,6 +94,19 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
85
94
  f"All batch labels must be non-null. Check your data for missing entries."
86
95
  )
87
96
 
97
+ if self.subject_id is not None:
98
+ subj_ser = (
99
+ pd.Series(self.subject_id)
100
+ if not isinstance(self.subject_id, pd.Series)
101
+ else self.subject_id
102
+ )
103
+ nan_count = int(subj_ser.isna().sum())
104
+ if nan_count:
105
+ raise ValueError(
106
+ f"subject_id contains {nan_count} NaN value(s). "
107
+ f"All subject labels must be non-null."
108
+ )
109
+
88
110
  if self.discrete_covariates is not None:
89
111
  disc_df = (
90
112
  pd.DataFrame(self.discrete_covariates)
@@ -118,6 +140,7 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
118
140
  batch_ser: pd.Series,
119
141
  disc: pd.DataFrame | pd.Series | None,
120
142
  cont: pd.DataFrame | pd.Series | None,
143
+ time: pd.DataFrame | pd.Series | None = None,
121
144
  ) -> None:
122
145
  """Issue warnings for data quality issues that may affect results."""
123
146
  # Near-zero variance features
@@ -148,8 +171,10 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
148
171
  stacklevel=3,
149
172
  )
150
173
 
151
- # Covariate collinearity with batch (fortin/chen only)
152
- if self.method.lower() in {"fortin", "chen"} and (disc is not None or cont is not None):
174
+ # Covariate collinearity with batch (fortin/chen/longitudinal only)
175
+ if self.method.lower() in {"fortin", "chen", "longitudinal"} and (
176
+ disc is not None or cont is not None or time is not None
177
+ ):
153
178
  batch_dummies = pd.get_dummies(batch_ser, drop_first=True).astype(float)
154
179
  cov_parts: list[pd.DataFrame] = []
155
180
  if disc is not None:
@@ -160,6 +185,9 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
160
185
  if cont is not None:
161
186
  c = cont if isinstance(cont, pd.DataFrame) else cont.to_frame()
162
187
  cov_parts.append(c.astype(float))
188
+ if time is not None:
189
+ t = time if isinstance(time, pd.DataFrame) else time.to_frame()
190
+ cov_parts.append(t.astype(float))
163
191
  cov_df = pd.concat(cov_parts, axis=1)
164
192
  design = pd.concat([batch_dummies, cov_df], axis=1)
165
193
  rank = la.matrix_rank(design.values)
@@ -227,14 +255,18 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
227
255
  batch_vec = _subset(self.batch, idx)
228
256
  disc = _subset(self.discrete_covariates, idx)
229
257
  cont = _subset(self.continuous_covariates, idx)
258
+ subj = _subset(self.subject_id, idx)
259
+ time = _subset(self.time_covariate, idx)
230
260
 
231
- self._check_data_quality(X_df, batch_vec, disc, cont) # type: ignore[arg-type]
261
+ self._check_data_quality(X_df, batch_vec, disc, cont, time) # type: ignore[arg-type]
232
262
 
233
263
  self._model.fit(
234
264
  X,
235
265
  batch=batch_vec, # type: ignore[arg-type]
236
266
  discrete_covariates=disc,
237
267
  continuous_covariates=cont,
268
+ subject_id=subj,
269
+ time_covariate=time,
238
270
  )
239
271
  self._fitted_batch = batch_vec
240
272
 
@@ -261,11 +293,15 @@ class ComBat(BaseEstimator, TransformerMixin): # type: ignore[misc]
261
293
  batch_vec = _subset(self.batch, idx)
262
294
  disc = _subset(self.discrete_covariates, idx)
263
295
  cont = _subset(self.continuous_covariates, idx)
296
+ subj = _subset(self.subject_id, idx)
297
+ time = _subset(self.time_covariate, idx)
264
298
  X_transformed = self._model.transform(
265
299
  X,
266
300
  batch=batch_vec, # type: ignore[arg-type]
267
301
  discrete_covariates=disc,
268
302
  continuous_covariates=cont,
303
+ subject_id=subj,
304
+ time_covariate=time,
269
305
  )
270
306
 
271
307
  batch_arr = np.asarray(batch_vec)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: combatlearn
3
- Version: 2.0.0
3
+ Version: 2.1.0
4
4
  Summary: Batch-effect harmonization for machine learning frameworks.
5
5
  Author-email: Ettore Rocchi <ettoreroc@gmail.com>
6
6
  License: MIT
@@ -27,6 +27,7 @@ Requires-Dist: ruff>=0.8; extra == "dev"
27
27
  Requires-Dist: pre-commit>=3.0; extra == "dev"
28
28
  Requires-Dist: mypy>=1.0; extra == "dev"
29
29
  Requires-Dist: pandas-stubs; extra == "dev"
30
+ Requires-Dist: statsmodels>=0.14; extra == "dev"
30
31
  Provides-Extra: docs
31
32
  Requires-Dist: sphinx>=7.0.0; extra == "docs"
32
33
  Requires-Dist: pydata-sphinx-theme>=0.15.0; extra == "docs"
@@ -49,10 +50,11 @@ Dynamic: license-file
49
50
 
50
51
  **combatlearn** makes the popular _ComBat_ (and _CovBat_) batch-effect correction algorithm available for use into machine learning frameworks. It lets you harmonise high-dimensional data inside a scikit-learn `Pipeline`, so that cross-validation and grid-search automatically take batch structure into account, **without data leakage**.
51
52
 
52
- **Three methods**:
53
+ **Four methods**:
53
54
  - `method="johnson"` - classic ComBat (Johnson _et al._, 2007)
54
55
  - `method="fortin"` - neuroComBat (Fortin _et al._, 2018)
55
56
  - `method="chen"` - CovBat (Chen _et al._, 2022)
57
+ - `method="longitudinal"` - Longitudinal ComBat for repeated measures (Beer _et al._, 2020)
56
58
 
57
59
  ## Installation
58
60
 
@@ -75,6 +77,7 @@ import pandas as pd
75
77
  from sklearn.pipeline import Pipeline
76
78
  from sklearn.preprocessing import StandardScaler
77
79
  from sklearn.linear_model import LogisticRegression
80
+ from sklearn.model_selection import GridSearchCV
78
81
  from combatlearn import ComBat
79
82
 
80
83
  df = pd.read_csv("data.csv", index_col=0)
@@ -125,18 +128,20 @@ The following section provides a detailed explanation of all parameters availabl
125
128
  | Parameter | Type | Default | Description |
126
129
  | --- | --- | --- | --- |
127
130
  | `batch` | array-like or pd.Series | **required** | Vector indicating batch assignment for each sample. This is used to estimate and remove batch effects. |
128
- | `discrete_covariates` | array-like, pd.Series, or pd.DataFrame | `None` | Optional categorical covariates (e.g., sex, site). Only used in `"fortin"` and `"chen"` methods. |
129
- | `continuous_covariates` | array-like, pd.Series or pd.DataFrame | `None` | Optional continuous covariates (e.g., age). Only used in `"fortin"` and `"chen"` methods. |
131
+ | `discrete_covariates` | array-like, pd.Series, or pd.DataFrame | `None` | Optional categorical covariates (e.g., sex, site). Only used in `"fortin"`, `"chen"`, and `"longitudinal"` methods. |
132
+ | `continuous_covariates` | array-like, pd.Series or pd.DataFrame | `None` | Optional continuous covariates (e.g., age). Only used in `"fortin"`, `"chen"`, and `"longitudinal"` methods. |
133
+ | `subject_id` | array-like or pd.Series | `None` | Subject/individual labels for the random intercept. **Required** for `method="longitudinal"`, ignored otherwise. |
134
+ | `time_covariate` | array-like or pd.Series | `None` | Optional continuous time variable for repeated measures. Only used in `"longitudinal"`. |
130
135
 
131
136
  ### Algorithm Options
132
137
 
133
138
  | Parameter | Type | Default | Description |
134
139
  | --- | --- | --- | --- |
135
- | `method` | str | `"johnson"` | ComBat method to use: <ul><li>`"johnson"` - Classical ComBat (_Johnson et al. 2007_)</li><li>`"fortin"` - ComBat with covariates (_Fortin et al. 2018_)</li><li>`"chen"` - CovBat, PCA-based correction (_Chen et al. 2022_)</li></ul> |
140
+ | `method` | str | `"johnson"` | ComBat method to use: <ul><li>`"johnson"` - Classical ComBat (_Johnson et al. 2007_)</li><li>`"fortin"` - ComBat with covariates (_Fortin et al. 2018_)</li><li>`"chen"` - CovBat, PCA-based correction (_Chen et al. 2022_)</li><li>`"longitudinal"` - Longitudinal ComBat with a per-subject random intercept (_Beer et al. 2020_)</li></ul> |
136
141
  | `parametric` | bool | `True` | Whether to use the **parametric empirical Bayes** formulation. If `False`, a non-parametric iterative scheme is used. |
137
142
  | `mean_only` | bool | `False` | If `True`, only the **mean** is corrected, while variances are left unchanged. Useful for preserving variance structure in the data. |
138
143
  | `reference_batch` | str or `None` | `None` | If specified, acts as a reference batch - other batches will be corrected to match this one. |
139
- | `covbat_cov_thresh` | float, int | `0.9` | For `"chen"` method only: Cumulative variance threshold $]0,1[$ to retain PCs in PCA space (e.g., 0.9 = retain 90% explained variance). If an integer is provided, it represents the number of principal components to use. |
144
+ | `covbat_cov_thresh` | float, int | `0.9` | For `"chen"` method only: Cumulative variance threshold in `(0, 1]` to retain PCs in PCA space (e.g., 0.9 = retain 90% explained variance). If an integer is provided, it represents the number of principal components to use. |
140
145
  | `eps` | float | `1e-8` | Small jitter value added to variances to prevent divide-by-zero errors during standardization. |
141
146
 
142
147
 
@@ -190,3 +195,5 @@ Please consider citing the original papers:
190
195
  - [**neuroCombat**](https://github.com/Jfortin1/neuroCombat) - Fortin JP et al. _Neuroimage_. 2018. doi: [10.1016/j.neuroimage.2017.11.024](https://doi.org/10.1016/j.neuroimage.2017.11.024)
191
196
 
192
197
  - [**CovBat**](https://github.com/andy1764/CovBat_Harmonization) - Chen AA et al. _Hum Brain Mapp_. 2022. doi: [10.1002/hbm.25688](https://doi.org/10.1002/hbm.25688)
198
+
199
+ - [**Longitudinal ComBat**](https://github.com/jcbeer/longCombat) - Beer JC et al. _Neuroimage_. 2020. doi: [10.1016/j.neuroimage.2020.117129](https://doi.org/10.1016/j.neuroimage.2020.117129)
@@ -5,6 +5,7 @@ requirements-dev.txt
5
5
  requirements-docs.txt
6
6
  requirements.txt
7
7
  combatlearn/__init__.py
8
+ combatlearn/_mixed.py
8
9
  combatlearn/_utils.py
9
10
  combatlearn/core.py
10
11
  combatlearn/inspection.py
@@ -19,5 +20,6 @@ combatlearn.egg-info/requires.txt
19
20
  combatlearn.egg-info/top_level.txt
20
21
  tests/test_combat.py
21
22
  tests/test_inspection.py
23
+ tests/test_longitudinal.py
22
24
  tests/test_metrics.py
23
25
  tests/test_visualization.py
@@ -14,6 +14,7 @@ ruff>=0.8
14
14
  pre-commit>=3.0
15
15
  mypy>=1.0
16
16
  pandas-stubs
17
+ statsmodels>=0.14
17
18
 
18
19
  [docs]
19
20
  sphinx>=7.0.0
@@ -4,3 +4,4 @@ ruff>=0.8
4
4
  pre-commit>=3.0
5
5
  mypy>=1.0
6
6
  pandas-stubs
7
+ statsmodels>=0.14
@@ -0,0 +1,204 @@
1
+ import numpy as np
2
+ import pandas as pd
3
+ import pytest
4
+ from sklearn.base import clone
5
+ from sklearn.pipeline import Pipeline
6
+ from sklearn.preprocessing import StandardScaler
7
+ from utils import simulate_longitudinal_data
8
+
9
+ from combatlearn import ComBat
10
+ from combatlearn._mixed import fit_random_intercept
11
+ from combatlearn.core import ComBatModel
12
+
13
+
14
+ def _fit_kwargs():
15
+ X, batch, subject, time = simulate_longitudinal_data()
16
+ return X, {
17
+ "batch": batch,
18
+ "subject_id": subject,
19
+ "time_covariate": time,
20
+ "method": "longitudinal",
21
+ }
22
+
23
+
24
+ def test_longitudinal_shape_index_columns_preserved():
25
+ """Output keeps shape, index, and column names."""
26
+ X, kwargs = _fit_kwargs()
27
+ X.index = pd.Index([f"obs_{i}" for i in range(len(X))])
28
+ kwargs["batch"].index = X.index
29
+ kwargs["subject_id"].index = X.index
30
+ kwargs["time_covariate"].index = X.index
31
+ X.columns = [f"feat_{i}" for i in range(X.shape[1])]
32
+
33
+ X_corr = ComBat(**kwargs).fit_transform(X)
34
+ assert X_corr.shape == X.shape
35
+ assert X_corr.index.equals(X.index)
36
+ assert list(X_corr.columns) == list(X.columns)
37
+
38
+
39
+ def test_longitudinal_no_nan_or_inf():
40
+ """Correction introduces no NaN/Inf."""
41
+ X, kwargs = _fit_kwargs()
42
+ X_corr = ComBat(**kwargs).fit_transform(X)
43
+ assert not np.isnan(X_corr.values).any()
44
+ assert not np.isinf(X_corr.values).any()
45
+
46
+
47
+ @pytest.mark.parametrize("mean_only", [True, False])
48
+ def test_longitudinal_mean_only(mean_only):
49
+ """mean_only mode works."""
50
+ X, kwargs = _fit_kwargs()
51
+ X_corr = ComBat(mean_only=mean_only, **kwargs).fit_transform(X)
52
+ assert X_corr.shape == X.shape
53
+ assert not np.isnan(X_corr.values).any()
54
+
55
+
56
+ @pytest.mark.parametrize("parametric", [True, False])
57
+ def test_longitudinal_parametric_flag(parametric):
58
+ """Both EB variants run."""
59
+ X, kwargs = _fit_kwargs()
60
+ X_corr = ComBat(parametric=parametric, **kwargs).fit_transform(X)
61
+ assert X_corr.shape == X.shape
62
+ assert not np.isnan(X_corr.values).any()
63
+
64
+
65
+ def test_longitudinal_reduces_batch_variance():
66
+ """Harmonization should reduce the fraction of variance explained by batch."""
67
+ X, kwargs = _fit_kwargs()
68
+ cb = ComBat(**kwargs).fit(X)
69
+ cb.transform(X)
70
+ assert cb._batch_var_after_ < cb._batch_var_before_
71
+
72
+
73
+ def test_longitudinal_requires_subject_id():
74
+ """method='longitudinal' without subject_id raises."""
75
+ X, batch, _subject, time = simulate_longitudinal_data()
76
+ with pytest.raises(ValueError, match=r"requires `subject_id`"):
77
+ ComBat(batch=batch, time_covariate=time, method="longitudinal").fit(X)
78
+
79
+
80
+ def test_subject_id_ignored_warning_for_other_methods():
81
+ """Passing subject_id with a non-longitudinal method warns that it is ignored."""
82
+ X, batch, subject, _time = simulate_longitudinal_data()
83
+ with pytest.warns(UserWarning, match="only used when"):
84
+ ComBat(batch=batch, subject_id=subject, method="johnson").fit(X)
85
+
86
+
87
+ def test_longitudinal_reference_batch_unchanged():
88
+ """Reference-batch samples come out numerically identical."""
89
+ X, batch, subject, time = simulate_longitudinal_data()
90
+ ref = batch.iloc[0]
91
+ cb = ComBat(
92
+ batch=batch,
93
+ subject_id=subject,
94
+ time_covariate=time,
95
+ method="longitudinal",
96
+ reference_batch=ref,
97
+ ).fit(X)
98
+ X_corr = cb.transform(X)
99
+ mask = (batch == ref).to_numpy()
100
+ np.testing.assert_allclose(X_corr.values[mask], X.values[mask], rtol=0, atol=1e-8)
101
+
102
+
103
+ def test_longitudinal_clone_and_pipeline():
104
+ """Wrapper clones and runs inside a Pipeline reproducibly."""
105
+ X, kwargs = _fit_kwargs()
106
+ pipe = Pipeline([("combat", ComBat(**kwargs)), ("scaler", StandardScaler())])
107
+ X_corr = pipe.fit_transform(X)
108
+ X_corr2 = clone(pipe).fit_transform(X)
109
+ np.testing.assert_allclose(X_corr, X_corr2, rtol=1e-6, atol=1e-6)
110
+
111
+
112
+ def test_longitudinal_unseen_subject_falls_back_to_zero_re():
113
+ """Transforming subjects unseen at fit (zero random intercept) must not crash."""
114
+ X, batch, subject, time = simulate_longitudinal_data()
115
+ model = ComBatModel(method="longitudinal").fit(
116
+ X, batch=batch, subject_id=subject, time_covariate=time
117
+ )
118
+ subject_new = subject.map(lambda s: f"new_{s}")
119
+ X_corr = model.transform(X, batch=batch, subject_id=subject_new, time_covariate=time)
120
+ assert X_corr.shape == X.shape
121
+ assert not np.isnan(X_corr.values).any()
122
+
123
+
124
+ def test_longitudinal_no_covariates():
125
+ """Longitudinal with only a subject random intercept (no covariates) works."""
126
+ X, batch, subject, _time = simulate_longitudinal_data()
127
+ X_corr = ComBat(batch=batch, subject_id=subject, method="longitudinal").fit_transform(X)
128
+ assert X_corr.shape == X.shape
129
+ assert not np.isnan(X_corr.values).any()
130
+
131
+
132
+ def test_longitudinal_collinear_discrete_covariate_warns():
133
+ """A discrete covariate collinear with batch should warn for the longitudinal method too."""
134
+ X, batch, subject, _time = simulate_longitudinal_data()
135
+ disc_collinear = batch.to_frame("cov")
136
+ with pytest.warns(UserWarning, match="rank-deficient"):
137
+ ComBat(
138
+ batch=batch,
139
+ subject_id=subject,
140
+ discrete_covariates=disc_collinear,
141
+ method="longitudinal",
142
+ ).fit(X)
143
+
144
+
145
+ def test_longitudinal_collinear_time_covariate_warns():
146
+ """A time covariate collinear with batch should also trigger the rank-deficiency warning."""
147
+ X, batch, subject, _time = simulate_longitudinal_data()
148
+ time_collinear = batch.map({"A": 0.0, "B": 1.0, "C": 2.0}).rename("visit")
149
+ with pytest.warns(UserWarning, match="rank-deficient"):
150
+ ComBat(
151
+ batch=batch,
152
+ subject_id=subject,
153
+ time_covariate=time_collinear,
154
+ method="longitudinal",
155
+ ).fit(X)
156
+
157
+
158
+ def test_random_intercept_recovers_fixed_effects():
159
+ """The REML fitter recovers known fixed effects on simulated data."""
160
+ rng = np.random.default_rng(0)
161
+ n_subjects, n_times = 60, 4
162
+ n = n_subjects * n_times
163
+ group_idx = np.repeat(np.arange(n_subjects), n_times).astype(np.intp)
164
+ n_k = np.bincount(group_idx).astype(np.float64)
165
+
166
+ x1 = rng.standard_normal(n)
167
+ beta_true = np.array([2.0, 1.5])
168
+ u = rng.normal(0.0, 1.0, n_subjects)
169
+ eps = rng.normal(0.0, 1.0, n)
170
+ y = beta_true[0] + beta_true[1] * x1 + u[group_idx] + eps
171
+
172
+ X = np.column_stack([np.ones(n), x1])
173
+ beta, _blup, sigma2, lam_opt = fit_random_intercept(X, y[:, None], group_idx, n_subjects, n_k)
174
+
175
+ np.testing.assert_allclose(beta[:, 0], beta_true, rtol=0, atol=0.25)
176
+ assert 0.5 < sigma2[0] < 2.0
177
+ assert lam_opt[0] > 0.0 # a real random intercept is detected
178
+
179
+
180
+ def test_random_intercept_matches_statsmodels():
181
+ """Cross-check REML estimates against statsmodels.MixedLM."""
182
+ sm = pytest.importorskip("statsmodels.api")
183
+
184
+ rng = np.random.default_rng(1)
185
+ n_subjects, n_times = 60, 5
186
+ n = n_subjects * n_times
187
+ group_idx = np.repeat(np.arange(n_subjects), n_times).astype(np.intp)
188
+ n_k = np.bincount(group_idx).astype(np.float64)
189
+
190
+ x1 = rng.standard_normal(n)
191
+ u = rng.normal(0.0, 1.2, n_subjects)
192
+ eps = rng.normal(0.0, 0.9, n)
193
+ y = 1.0 + 0.8 * x1 + u[group_idx] + eps
194
+
195
+ X = np.column_stack([np.ones(n), x1])
196
+ beta, _blup, sigma2, lam_opt = fit_random_intercept(X, y[:, None], group_idx, n_subjects, n_k)
197
+ tau2 = lam_opt[0] * sigma2[0]
198
+
199
+ res = sm.MixedLM(endog=y, exog=X, groups=group_idx).fit(reml=True, method="lbfgs")
200
+ sm_tau2 = float(np.asarray(res.cov_re)[0, 0])
201
+
202
+ np.testing.assert_allclose(beta[:, 0], res.fe_params, rtol=0.02, atol=0.02)
203
+ np.testing.assert_allclose(sigma2[0], res.scale, rtol=0.05)
204
+ np.testing.assert_allclose(tau2, sm_tau2, rtol=0.12, atol=0.05)
File without changes
File without changes
File without changes