MaldiBatchKit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- maldibatchkit/__init__.py +56 -0
- maldibatchkit/_base.py +176 -0
- maldibatchkit/_cli_io.py +211 -0
- maldibatchkit/_utils.py +122 -0
- maldibatchkit/cli.py +497 -0
- maldibatchkit/corrections/__init__.py +20 -0
- maldibatchkit/corrections/baselines.py +235 -0
- maldibatchkit/corrections/combat.py +84 -0
- maldibatchkit/corrections/harmony.py +445 -0
- maldibatchkit/corrections/limma.py +165 -0
- maldibatchkit/corrections/maldi.py +167 -0
- maldibatchkit/corrections/quality_weighted.py +338 -0
- maldibatchkit/diagnostics/__init__.py +39 -0
- maldibatchkit/diagnostics/generic.py +327 -0
- maldibatchkit/diagnostics/maldi.py +165 -0
- maldibatchkit/diagnostics/report.py +151 -0
- maldibatchkit/integrations/__init__.py +9 -0
- maldibatchkit/integrations/maldiset.py +184 -0
- maldibatchkit/py.typed +0 -0
- maldibatchkit/viz/__init__.py +11 -0
- maldibatchkit/viz/peaks.py +82 -0
- maldibatchkit/viz/summary.py +155 -0
- maldibatchkit/viz/umap.py +118 -0
- maldibatchkit-0.1.0.dist-info/METADATA +344 -0
- maldibatchkit-0.1.0.dist-info/RECORD +29 -0
- maldibatchkit-0.1.0.dist-info/WHEEL +5 -0
- maldibatchkit-0.1.0.dist-info/entry_points.txt +2 -0
- maldibatchkit-0.1.0.dist-info/licenses/LICENSE +21 -0
- maldibatchkit-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,445 @@
|
|
|
1
|
+
r"""Harmony integration (Korsunsky et al. 2019) via ``harmonypy``.
|
|
2
|
+
|
|
3
|
+
Harmony is iterative and has no closed-form "transform on new data"
|
|
4
|
+
step in its published form. Concretely, its fitted state consists of:
|
|
5
|
+
|
|
6
|
+
* L2-normalised cluster centroids ``Y`` (``d × K``),
|
|
7
|
+
* a soft-assignment bandwidth ``sigma`` (``K,``),
|
|
8
|
+
* a per-cluster ridge-regression batch correction ``W_batch``
|
|
9
|
+
(``K × B × d``) that was applied to the training rows during the
|
|
10
|
+
last ``moe_correct_ridge`` call.
|
|
11
|
+
|
|
12
|
+
Given those, the correction for a *new* sample reduces to a closed
|
|
13
|
+
form:
|
|
14
|
+
|
|
15
|
+
.. math::
|
|
16
|
+
|
|
17
|
+
r_{ik} &= \\mathrm{softmax}_k(-\\lVert \\hat{x}_i - Y_k
|
|
18
|
+
\\rVert^2 / \\sigma_k)
|
|
19
|
+
|
|
20
|
+
\\hat{x}_i &= x_i / \\lVert x_i \\rVert_2
|
|
21
|
+
|
|
22
|
+
x_i^{\\text{corr}} &= x_i - \\sum_k r_{ik}\\, W_{\\text{batch}}^{\\,k,\\,
|
|
23
|
+
b(i)}
|
|
24
|
+
|
|
25
|
+
where :math:`b(i)` is sample :math:`i`'s batch index. This mirrors
|
|
26
|
+
what Harmony does to training rows at its last iteration, but with
|
|
27
|
+
:math:`Y` and :math:`W_{\\text{batch}}` frozen at fit time, so test
|
|
28
|
+
rows cannot leak into the correction.
|
|
29
|
+
|
|
30
|
+
For the common case where ``fit`` and ``transform`` receive the same
|
|
31
|
+
rows (``fit_transform`` or simply re-``transform``), we short-circuit
|
|
32
|
+
to the exact ``harmonypy`` output so users see bit-identical behaviour
|
|
33
|
+
to the upstream library.
|
|
34
|
+
|
|
35
|
+
Built-in PCA preprocessing
|
|
36
|
+
--------------------------
|
|
37
|
+
Harmony was designed for low-dimensional dense embeddings (~20-50 PCA
|
|
38
|
+
components). Applying it directly to a 6000-bin MALDI-TOF intensity
|
|
39
|
+
matrix degrades the soft-cluster assignment, because cosine distances
|
|
40
|
+
in the raw feature space are dominated by a handful of high-intensity
|
|
41
|
+
bins. To sidestep that, :class:`Harmony` **always** fits a ``PCA``
|
|
42
|
+
on the training rows at ``fit`` time, runs Harmony in PCA space, and
|
|
43
|
+
maps the corrected result back to the original feature space via
|
|
44
|
+
``PCA.inverse_transform``. The PCA itself is **frozen** at ``fit``
|
|
45
|
+
(like :math:`Y` and :math:`W_{\\text{batch}}`) and re-used verbatim
|
|
46
|
+
on unseen rows, so the closed-form transform stays leakage-safe by
|
|
47
|
+
construction. Use the ``n_components`` argument to tune the size of
|
|
48
|
+
the PCA basis; ``None`` (the default) picks ``min(50, n_samples - 1,
|
|
49
|
+
n_features)``.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
from __future__ import annotations
|
|
53
|
+
|
|
54
|
+
import contextlib
|
|
55
|
+
import logging
|
|
56
|
+
from typing import Any
|
|
57
|
+
|
|
58
|
+
import numpy as np
|
|
59
|
+
import numpy.typing as npt
|
|
60
|
+
import pandas as pd
|
|
61
|
+
from sklearn.decomposition import PCA
|
|
62
|
+
from sklearn.preprocessing import StandardScaler
|
|
63
|
+
|
|
64
|
+
from .._base import BaseBatchCorrector
|
|
65
|
+
from .._utils import ArrayLike
|
|
66
|
+
|
|
67
|
+
_PCA_DEFAULT_N_COMPONENTS = 50
|
|
68
|
+
|
|
69
|
+
__all__ = ["Harmony"]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@contextlib.contextmanager
|
|
73
|
+
def _silence_harmonypy_logger():
|
|
74
|
+
"""Temporarily raise the ``harmonypy`` logger above INFO.
|
|
75
|
+
|
|
76
|
+
``harmonypy`` configures its own logger at module-import time with a
|
|
77
|
+
DEBUG level and a StreamHandler attached, so setting ``verbose=False``
|
|
78
|
+
on ``run_harmony`` alone does not suppress its progress messages.
|
|
79
|
+
This context manager elevates the logger to WARNING for the duration
|
|
80
|
+
of the call and restores the original state afterwards, leaving the
|
|
81
|
+
user's broader logging configuration untouched.
|
|
82
|
+
"""
|
|
83
|
+
logger = logging.getLogger("harmonypy")
|
|
84
|
+
prev_level = logger.level
|
|
85
|
+
prev_propagate = logger.propagate
|
|
86
|
+
logger.setLevel(logging.WARNING)
|
|
87
|
+
logger.propagate = False
|
|
88
|
+
try:
|
|
89
|
+
yield
|
|
90
|
+
finally:
|
|
91
|
+
logger.setLevel(prev_level)
|
|
92
|
+
logger.propagate = prev_propagate
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class Harmony(BaseBatchCorrector):
|
|
96
|
+
"""Sklearn-compatible Harmony with a closed-form ``transform``.
|
|
97
|
+
|
|
98
|
+
Parameters
|
|
99
|
+
----------
|
|
100
|
+
batch : array-like of shape (n_samples,)
|
|
101
|
+
Batch labels.
|
|
102
|
+
covariates : array-like, optional
|
|
103
|
+
Additional categorical covariates forwarded to
|
|
104
|
+
``harmonypy.run_harmony`` via the ``vars_use`` mechanism.
|
|
105
|
+
theta : float or list of float, default=2.0
|
|
106
|
+
Diversity clustering penalty in Harmony. Higher values
|
|
107
|
+
encourage more aggressive batch mixing.
|
|
108
|
+
max_iter : int, default=20
|
|
109
|
+
Harmony iteration cap.
|
|
110
|
+
nclust : int, optional
|
|
111
|
+
Number of Harmony clusters. When ``None``, ``harmonypy``'s
|
|
112
|
+
default heuristic is used.
|
|
113
|
+
n_components : int, optional
|
|
114
|
+
Number of PCA components used for the built-in dimensionality-
|
|
115
|
+
reduction stage. ``None`` (the default) picks
|
|
116
|
+
``min(50, n_samples - 1, n_features)``.
|
|
117
|
+
random_state : int, optional
|
|
118
|
+
Seed forwarded to harmonypy **and** to the PCA stage.
|
|
119
|
+
verbose : bool, default=True
|
|
120
|
+
If ``False``, suppress ``harmonypy``'s own progress messages
|
|
121
|
+
during ``fit``. ``harmonypy`` configures its module-level
|
|
122
|
+
logger eagerly at import time, so simply passing
|
|
123
|
+
``verbose=False`` to ``run_harmony`` is not enough; this flag
|
|
124
|
+
also raises the ``harmonypy`` logger above INFO for the
|
|
125
|
+
duration of the call. The user's broader logging config is
|
|
126
|
+
left untouched.
|
|
127
|
+
scale_before_pca : bool, default=True
|
|
128
|
+
If ``True`` (the default), fit a :class:`~sklearn.preprocessing.StandardScaler`
|
|
129
|
+
on the training rows and apply it before the internal PCA. This
|
|
130
|
+
gives every feature equal weight in the principal components and
|
|
131
|
+
matches the convention from single-cell genomics workflows
|
|
132
|
+
(Seurat / Scanpy ``ScaleData`` followed by PCA), where Harmony
|
|
133
|
+
was originally developed. Without scaling, MALDI-TOF features
|
|
134
|
+
with high intensity (a handful of intense peaks) dominate the
|
|
135
|
+
PCs and Harmony's soft-cluster geometry inherits that bias.
|
|
136
|
+
The scaler is **frozen** at ``fit`` (like the PCA, the centroids
|
|
137
|
+
``Y``, and the per-batch ridge ``W_batch``) and reused verbatim
|
|
138
|
+
on unseen rows at ``transform`` time, with the corrected output
|
|
139
|
+
un-scaled back to the original measurement scale. Set to
|
|
140
|
+
``False`` to recover the pre-v0.2 behaviour (centred but not
|
|
141
|
+
scaled before PCA).
|
|
142
|
+
|
|
143
|
+
Attributes
|
|
144
|
+
----------
|
|
145
|
+
batch_levels_ : np.ndarray
|
|
146
|
+
Batch levels in the order used to encode ``Phi_moe`` columns.
|
|
147
|
+
Y_ : np.ndarray of shape (n_components, K)
|
|
148
|
+
L2-normalised cluster centroids frozen at fit time (in PCA
|
|
149
|
+
space).
|
|
150
|
+
sigma_ : np.ndarray of shape (K,)
|
|
151
|
+
Soft-assignment bandwidths per cluster.
|
|
152
|
+
W_batch_ : np.ndarray of shape (K, B, n_components)
|
|
153
|
+
Per-cluster, per-batch ridge-regression correction vectors, in
|
|
154
|
+
the same PCA space as ``Y_``.
|
|
155
|
+
pca_ : sklearn.decomposition.PCA
|
|
156
|
+
Fitted PCA, re-used verbatim on unseen rows at transform time.
|
|
157
|
+
X_fit_ : np.ndarray
|
|
158
|
+
Training feature matrix (original space) cached for
|
|
159
|
+
same-matrix short-circuiting.
|
|
160
|
+
n_clusters_ : int
|
|
161
|
+
Number of clusters actually used by Harmony.
|
|
162
|
+
n_iter_ : int
|
|
163
|
+
Harmony iterations observed at fit time.
|
|
164
|
+
|
|
165
|
+
Notes
|
|
166
|
+
-----
|
|
167
|
+
* The closed-form output on ``X_train`` is not quite bit-identical
|
|
168
|
+
to ``harmonypy``'s iterative correction because Harmony's soft
|
|
169
|
+
assignment is refined by a batch-diversity penalty during
|
|
170
|
+
training (the ``update_R`` loop). We deliberately use the raw
|
|
171
|
+
cosine-softmax assignment at transform time since the diversity
|
|
172
|
+
correction is a *training-time regulariser* tied to the training
|
|
173
|
+
batch distribution and would not apply to a held-out sample.
|
|
174
|
+
When ``transform`` is called on the exact training matrix we
|
|
175
|
+
short-circuit to the harmonypy output so that ``fit_transform``
|
|
176
|
+
matches upstream byte-for-byte.
|
|
177
|
+
* Unseen batch levels at transform raise ``ValueError``, mirroring
|
|
178
|
+
``ComBat`` / ``Limma``.
|
|
179
|
+
* Requires ``harmonypy``; it is a core dependency of
|
|
180
|
+
MaldiBatchKit, range-pinned to ``>=0.2.0,<2``. The 2.x line is
|
|
181
|
+
a C++ rewrite that drops the private attributes our closed-form
|
|
182
|
+
transform relies on (``_Phi_moe``, ``_lamb``, ``sigma``); we
|
|
183
|
+
will revisit this constraint once those are re-exposed upstream.
|
|
184
|
+
|
|
185
|
+
Examples
|
|
186
|
+
--------
|
|
187
|
+
>>> from maldibatchkit import Harmony
|
|
188
|
+
>>> corrector = Harmony(batch=batches, random_state=0)
|
|
189
|
+
>>> X_train_c = corrector.fit_transform(X_train)
|
|
190
|
+
>>> X_test_c = corrector.transform(X_test) # closed-form on new rows
|
|
191
|
+
"""
|
|
192
|
+
|
|
193
|
+
def __init__(
|
|
194
|
+
self,
|
|
195
|
+
batch: ArrayLike,
|
|
196
|
+
*,
|
|
197
|
+
covariates: ArrayLike | None = None,
|
|
198
|
+
theta: float | list[float] = 2.0,
|
|
199
|
+
max_iter: int = 20,
|
|
200
|
+
nclust: int | None = None,
|
|
201
|
+
n_components: int | None = None,
|
|
202
|
+
random_state: int | None = None,
|
|
203
|
+
verbose: bool = True,
|
|
204
|
+
scale_before_pca: bool = True,
|
|
205
|
+
) -> None:
|
|
206
|
+
super().__init__(batch=batch)
|
|
207
|
+
self.covariates = covariates
|
|
208
|
+
self.theta = theta
|
|
209
|
+
self.max_iter = max_iter
|
|
210
|
+
self.nclust = nclust
|
|
211
|
+
self.n_components = n_components
|
|
212
|
+
self.random_state = random_state
|
|
213
|
+
self.verbose = verbose
|
|
214
|
+
self.scale_before_pca = scale_before_pca
|
|
215
|
+
|
|
216
|
+
def _resolve_n_components(self, n_samples: int, n_features: int) -> int:
|
|
217
|
+
"""Pick a safe number of PCs for a given (n_samples, n_features)."""
|
|
218
|
+
cap = max(1, min(n_samples - 1, n_features))
|
|
219
|
+
if self.n_components is None:
|
|
220
|
+
return min(_PCA_DEFAULT_N_COMPONENTS, cap)
|
|
221
|
+
return int(min(self.n_components, cap))
|
|
222
|
+
|
|
223
|
+
@staticmethod
|
|
224
|
+
def _require_harmonypy():
|
|
225
|
+
try:
|
|
226
|
+
import harmonypy # noqa: F401
|
|
227
|
+
except ImportError as exc:
|
|
228
|
+
raise ImportError(
|
|
229
|
+
"Harmony requires the optional 'harmonypy' "
|
|
230
|
+
"dependency. Install it with `pip install harmonypy` "
|
|
231
|
+
"(it is pulled in automatically by `pip install maldibatchkit`)."
|
|
232
|
+
) from exc
|
|
233
|
+
return harmonypy
|
|
234
|
+
|
|
235
|
+
def _build_meta(
|
|
236
|
+
self, idx: pd.Index, batch: npt.NDArray[Any]
|
|
237
|
+
) -> tuple[pd.DataFrame, list[str]]:
|
|
238
|
+
"""Construct the harmonypy ``meta_data`` DataFrame + ``vars_use`` list."""
|
|
239
|
+
meta = pd.DataFrame(
|
|
240
|
+
{"batch": np.asarray(batch, dtype=object).astype(str)},
|
|
241
|
+
index=idx,
|
|
242
|
+
)
|
|
243
|
+
vars_use = ["batch"]
|
|
244
|
+
if self.covariates is not None:
|
|
245
|
+
cov = self.covariates
|
|
246
|
+
if isinstance(cov, pd.Series):
|
|
247
|
+
cov = cov.to_frame()
|
|
248
|
+
if not isinstance(cov, pd.DataFrame):
|
|
249
|
+
cov = pd.DataFrame(np.asarray(cov), index=idx)
|
|
250
|
+
cov_aligned = cov.loc[idx].copy().astype(str)
|
|
251
|
+
cov_aligned.columns = [f"cov_{i}" for i in range(cov_aligned.shape[1])]
|
|
252
|
+
meta = pd.concat([meta, cov_aligned], axis=1)
|
|
253
|
+
vars_use.extend(list(cov_aligned.columns))
|
|
254
|
+
return meta, vars_use
|
|
255
|
+
|
|
256
|
+
@staticmethod
|
|
257
|
+
def _l2_normalise_rows(X: np.ndarray, eps: float = 1e-12) -> np.ndarray:
|
|
258
|
+
norms = np.linalg.norm(X, axis=1, keepdims=True)
|
|
259
|
+
norms = np.clip(norms, eps, None)
|
|
260
|
+
return X / norms
|
|
261
|
+
|
|
262
|
+
def _softmax_assignment(self, X: np.ndarray) -> np.ndarray:
|
|
263
|
+
"""Compute the frozen-centroid soft assignment R (N, K)."""
|
|
264
|
+
Z_cos = self._l2_normalise_rows(X)
|
|
265
|
+
dist = 2.0 * (1.0 - Z_cos @ self.Y_) # (N, K)
|
|
266
|
+
scaled = -dist / self.sigma_[None, :]
|
|
267
|
+
scaled -= scaled.max(axis=1, keepdims=True) # numerical stability
|
|
268
|
+
R = np.exp(scaled)
|
|
269
|
+
R = R / np.clip(R.sum(axis=1, keepdims=True), 1e-12, None)
|
|
270
|
+
return R
|
|
271
|
+
|
|
272
|
+
@staticmethod
|
|
273
|
+
def _extract_W_batch(harmony_out, batch_levels: np.ndarray) -> np.ndarray:
|
|
274
|
+
"""Recompute the final-iteration per-cluster per-batch ridge W.
|
|
275
|
+
|
|
276
|
+
Harmony's ``Phi_moe`` has shape ``(1 + ΣL_v, N)`` where ``L_v``
|
|
277
|
+
is the number of levels of each variable in ``vars_use``. The
|
|
278
|
+
batch indicator occupies **the first** level block (rows
|
|
279
|
+
``1 : 1 + n_batches``), because ``_build_meta`` always places
|
|
280
|
+
``"batch"`` first in ``vars_use``. Additional covariates
|
|
281
|
+
contribute further level blocks that are *protected* in the
|
|
282
|
+
training correction but are not reapplied to unseen rows at
|
|
283
|
+
transform time - the closed-form path removes batch only.
|
|
284
|
+
|
|
285
|
+
Returns an array of shape ``(K, n_batches, d)`` such that
|
|
286
|
+
``W_batch[k, b, :]`` is the per-cluster, per-batch coefficient
|
|
287
|
+
subtracted for training samples that belong to batch ``b``.
|
|
288
|
+
"""
|
|
289
|
+
Z_orig = np.asarray(harmony_out._Z_orig.cpu().numpy(), dtype=float) # (d, N)
|
|
290
|
+
Phi_moe = np.asarray(
|
|
291
|
+
harmony_out._Phi_moe.cpu().numpy(), dtype=float
|
|
292
|
+
) # (rows, N)
|
|
293
|
+
R = np.asarray(harmony_out._R.cpu().numpy(), dtype=float) # (K, N)
|
|
294
|
+
lamb = np.asarray(harmony_out._lamb.cpu().numpy(), dtype=float) # (rows,)
|
|
295
|
+
|
|
296
|
+
K, _ = R.shape
|
|
297
|
+
d = Z_orig.shape[0]
|
|
298
|
+
n_batches = int(len(batch_levels))
|
|
299
|
+
|
|
300
|
+
W_batch = np.zeros((K, n_batches, d), dtype=float)
|
|
301
|
+
batch_slice = slice(1, 1 + n_batches) # skip intercept, take batch block
|
|
302
|
+
for k in range(K):
|
|
303
|
+
Phi_Rk = Phi_moe * R[k] # (rows, N)
|
|
304
|
+
cov = Phi_Rk @ Phi_moe.T + np.diag(lamb) # (rows, rows)
|
|
305
|
+
inv_cov = np.linalg.inv(cov)
|
|
306
|
+
Z_tmp = Z_orig * R[k] # (d, N)
|
|
307
|
+
rhs = Phi_moe @ Z_tmp.T # (rows, d)
|
|
308
|
+
W_k = inv_cov @ rhs # (rows, d)
|
|
309
|
+
W_k[0, :] = 0.0 # keep the intercept
|
|
310
|
+
W_batch[k] = W_k[batch_slice, :]
|
|
311
|
+
return W_batch
|
|
312
|
+
|
|
313
|
+
def _extract_batch_levels(self, harmony_out, batch: npt.NDArray[Any]) -> np.ndarray:
|
|
314
|
+
"""Extract the batch-level ordering implied by Phi_moe's columns.
|
|
315
|
+
|
|
316
|
+
``harmonypy`` uses ``pd.factorize(meta[vars_use[0]])`` under the
|
|
317
|
+
hood, so the order is the first-occurrence order in the meta
|
|
318
|
+
DataFrame we provided.
|
|
319
|
+
"""
|
|
320
|
+
seen: list[str] = []
|
|
321
|
+
for b in batch.tolist():
|
|
322
|
+
s = str(b)
|
|
323
|
+
if s not in seen:
|
|
324
|
+
seen.append(s)
|
|
325
|
+
return np.asarray(seen, dtype=object)
|
|
326
|
+
|
|
327
|
+
def _fit_impl(self, X_df: pd.DataFrame, batch: npt.NDArray[Any]) -> None:
|
|
328
|
+
harmonypy = self._require_harmonypy()
|
|
329
|
+
if self.random_state is not None:
|
|
330
|
+
np.random.seed(self.random_state)
|
|
331
|
+
|
|
332
|
+
meta, vars_use = self._build_meta(X_df.index, batch)
|
|
333
|
+
X_orig = X_df.to_numpy(dtype=float)
|
|
334
|
+
|
|
335
|
+
# Optional StandardScaler before PCA: gives every feature equal
|
|
336
|
+
# weight in the principal components, matching the Seurat /
|
|
337
|
+
# Scanpy ``ScaleData -> PCA`` convention from single-cell
|
|
338
|
+
# genomics where Harmony was first developed. The scaler is
|
|
339
|
+
# frozen at fit time and reused on unseen rows at transform.
|
|
340
|
+
if self.scale_before_pca:
|
|
341
|
+
self.scaler_ = StandardScaler().fit(X_orig)
|
|
342
|
+
X_scaled = self.scaler_.transform(X_orig)
|
|
343
|
+
else:
|
|
344
|
+
self.scaler_ = None
|
|
345
|
+
X_scaled = X_orig
|
|
346
|
+
|
|
347
|
+
# PCA preprocessing is mandatory and fit on training rows only.
|
|
348
|
+
# It is then frozen and re-used verbatim by ``transform`` so no
|
|
349
|
+
# information leaks from test rows back into the basis.
|
|
350
|
+
n_pcs = self._resolve_n_components(X_scaled.shape[0], X_scaled.shape[1])
|
|
351
|
+
self.pca_ = PCA(n_components=n_pcs, random_state=self.random_state)
|
|
352
|
+
X_work = self.pca_.fit_transform(X_scaled)
|
|
353
|
+
|
|
354
|
+
kwargs: dict[str, Any] = {
|
|
355
|
+
"vars_use": vars_use,
|
|
356
|
+
"theta": self.theta,
|
|
357
|
+
"max_iter_harmony": self.max_iter,
|
|
358
|
+
"verbose": self.verbose,
|
|
359
|
+
}
|
|
360
|
+
if self.nclust is not None:
|
|
361
|
+
kwargs["nclust"] = self.nclust
|
|
362
|
+
|
|
363
|
+
log_ctx = (
|
|
364
|
+
_silence_harmonypy_logger()
|
|
365
|
+
if not self.verbose
|
|
366
|
+
else contextlib.nullcontext()
|
|
367
|
+
)
|
|
368
|
+
with log_ctx:
|
|
369
|
+
harmony_out = harmonypy.run_harmony(X_work, meta, **kwargs)
|
|
370
|
+
|
|
371
|
+
self.batch_levels_ = self._extract_batch_levels(harmony_out, batch)
|
|
372
|
+
# Y_ / W_batch_ live in PCA space. The closed-form transform
|
|
373
|
+
# projects new rows into the same space before applying them.
|
|
374
|
+
self.Y_ = harmony_out.Y # (n_components, K)
|
|
375
|
+
self.sigma_ = harmony_out.sigma.astype(float) # (K,)
|
|
376
|
+
self.W_batch_ = self._extract_W_batch(
|
|
377
|
+
harmony_out, self.batch_levels_
|
|
378
|
+
) # (K, B, n_components)
|
|
379
|
+
self.n_clusters_ = int(self.Y_.shape[1])
|
|
380
|
+
self.n_iter_ = int(len(getattr(harmony_out, "kmeans_rounds", []) or [0]))
|
|
381
|
+
|
|
382
|
+
# Same-matrix fast-path cache in the original feature space -
|
|
383
|
+
# map harmonypy's Z_corr back through the fitted PCA so users
|
|
384
|
+
# who call ``transform(X_train)`` (or ``fit_transform``) see
|
|
385
|
+
# ``(n_samples, n_features)``. When the scaler is in use we
|
|
386
|
+
# also un-standardise so the cached output is on the original
|
|
387
|
+
# measurement scale, matching the closed-form transform path.
|
|
388
|
+
Z_corr_work = np.asarray(harmony_out.Z_corr) # (N, n_components)
|
|
389
|
+
Z_corr_scaled = self.pca_.inverse_transform(Z_corr_work)
|
|
390
|
+
if self.scaler_ is not None:
|
|
391
|
+
Z_corr_orig = self.scaler_.inverse_transform(Z_corr_scaled)
|
|
392
|
+
else:
|
|
393
|
+
Z_corr_orig = Z_corr_scaled
|
|
394
|
+
self.X_fit_ = X_orig
|
|
395
|
+
self._corrected_cache_ = Z_corr_orig # (N, n_features)
|
|
396
|
+
|
|
397
|
+
def _transform_impl(
|
|
398
|
+
self, X_df: pd.DataFrame, batch: npt.NDArray[Any]
|
|
399
|
+
) -> pd.DataFrame:
|
|
400
|
+
X_orig = X_df.to_numpy(dtype=float)
|
|
401
|
+
|
|
402
|
+
# Fast path: caller transforms the exact matrix we fit on ->
|
|
403
|
+
# return the cached correction byte-for-byte.
|
|
404
|
+
if X_orig.shape == self.X_fit_.shape and np.array_equal(X_orig, self.X_fit_):
|
|
405
|
+
return pd.DataFrame(
|
|
406
|
+
self._corrected_cache_,
|
|
407
|
+
index=X_df.index,
|
|
408
|
+
columns=X_df.columns,
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
# Closed-form correction on new rows --------------------------
|
|
412
|
+
batch_str = np.asarray([str(b) for b in batch])
|
|
413
|
+
known = {lvl: i for i, lvl in enumerate(self.batch_levels_)}
|
|
414
|
+
unseen = sorted(set(batch_str) - set(known))
|
|
415
|
+
if unseen:
|
|
416
|
+
raise ValueError(
|
|
417
|
+
f"Unseen batch level(s) at transform: {unseen}. "
|
|
418
|
+
f"Harmony's closed-form correction is only defined for "
|
|
419
|
+
f"batches that were present at fit time."
|
|
420
|
+
)
|
|
421
|
+
batch_idx = np.asarray([known[b] for b in batch_str], dtype=int)
|
|
422
|
+
|
|
423
|
+
# Project into PCA space using the fitted basis - no data from
|
|
424
|
+
# ``X_orig`` flows back into the scaler / PCA at fit time, so
|
|
425
|
+
# the train/test split stays clean.
|
|
426
|
+
if self.scaler_ is not None:
|
|
427
|
+
X_scaled = self.scaler_.transform(X_orig)
|
|
428
|
+
else:
|
|
429
|
+
X_scaled = X_orig
|
|
430
|
+
X_work = self.pca_.transform(X_scaled)
|
|
431
|
+
|
|
432
|
+
R_new = self._softmax_assignment(X_work) # (N, K)
|
|
433
|
+
|
|
434
|
+
# W_batch_ shape (K, B, n_components); pull the batch slice per
|
|
435
|
+
# sample -> (N, K, n_components); weight by R_new and sum over K.
|
|
436
|
+
W_selected = self.W_batch_[:, batch_idx, :].transpose(1, 0, 2)
|
|
437
|
+
correction = np.einsum("nk,nkd->nd", R_new, W_selected)
|
|
438
|
+
|
|
439
|
+
Z_corr_work = X_work - correction
|
|
440
|
+
Z_corr_scaled = self.pca_.inverse_transform(Z_corr_work)
|
|
441
|
+
if self.scaler_ is not None:
|
|
442
|
+
Z_corr_orig = self.scaler_.inverse_transform(Z_corr_scaled)
|
|
443
|
+
else:
|
|
444
|
+
Z_corr_orig = Z_corr_scaled
|
|
445
|
+
return pd.DataFrame(Z_corr_orig, index=X_df.index, columns=X_df.columns)
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
r"""Limma-style ``removeBatchEffect`` (Ritchie et al. 2015).
|
|
2
|
+
|
|
3
|
+
This is a faithful port of the linear-model batch subtraction performed by
|
|
4
|
+
``limma::removeBatchEffect``. Unlike ComBat, Limma makes no empirical-Bayes
|
|
5
|
+
shrinkage; it simply fits the batch indicators (plus any protected
|
|
6
|
+
covariates) by ordinary least squares and subtracts the fitted batch
|
|
7
|
+
contribution from the data.
|
|
8
|
+
|
|
9
|
+
Mathematically, for each feature we solve
|
|
10
|
+
|
|
11
|
+
.. math::
|
|
12
|
+
|
|
13
|
+
X = \\alpha + B \\gamma + C \\beta + \\varepsilon
|
|
14
|
+
|
|
15
|
+
where :math:`B` holds sum-to-zero contrasts on the batch indicator,
|
|
16
|
+
:math:`C` holds the protected covariates (design-of-interest), and we return
|
|
17
|
+
:math:`X - B \\hat\\gamma`. The intercept and covariate terms are left
|
|
18
|
+
untouched so biological signal encoded in ``design`` is preserved.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
import numpy as np
|
|
26
|
+
import numpy.linalg as la
|
|
27
|
+
import numpy.typing as npt
|
|
28
|
+
import pandas as pd
|
|
29
|
+
|
|
30
|
+
from .._base import BaseBatchCorrector
|
|
31
|
+
from .._utils import ArrayLike, _subset
|
|
32
|
+
|
|
33
|
+
__all__ = ["Limma"]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _sum_to_zero_contrasts(levels: np.ndarray) -> np.ndarray:
|
|
37
|
+
"""Return the ``(n_levels, n_levels - 1)`` sum-to-zero contrast matrix."""
|
|
38
|
+
n = len(levels)
|
|
39
|
+
if n < 2:
|
|
40
|
+
return np.zeros((n, 0))
|
|
41
|
+
contrasts = np.eye(n)[:, :-1]
|
|
42
|
+
contrasts[-1, :] = -1.0
|
|
43
|
+
return contrasts
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _design_matrix(
|
|
47
|
+
labels: np.ndarray, levels: np.ndarray
|
|
48
|
+
) -> tuple[np.ndarray, np.ndarray]:
|
|
49
|
+
"""Build a sum-to-zero batch design matrix aligned to ``labels``."""
|
|
50
|
+
contrasts = _sum_to_zero_contrasts(levels)
|
|
51
|
+
index_of = {lvl: i for i, lvl in enumerate(levels)}
|
|
52
|
+
unknown = [lvl for lvl in np.unique(labels) if lvl not in index_of]
|
|
53
|
+
if unknown:
|
|
54
|
+
raise ValueError(
|
|
55
|
+
f"Unseen batch level(s) at transform: {unknown}. "
|
|
56
|
+
f"Limma's batch subtraction is undefined for batches not "
|
|
57
|
+
f"present at fit time."
|
|
58
|
+
)
|
|
59
|
+
row_idx = np.asarray([index_of[lvl] for lvl in labels])
|
|
60
|
+
design = contrasts[row_idx]
|
|
61
|
+
return design, contrasts
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class Limma(BaseBatchCorrector):
|
|
65
|
+
"""Linear-model batch subtraction following ``limma::removeBatchEffect``.
|
|
66
|
+
|
|
67
|
+
Parameters
|
|
68
|
+
----------
|
|
69
|
+
batch : array-like of shape (n_samples,)
|
|
70
|
+
Batch labels.
|
|
71
|
+
design : array-like of shape (n_samples, n_covariates), optional
|
|
72
|
+
Protected covariates (the "design of interest" in Limma). Any
|
|
73
|
+
effect explained by ``design`` is left in the output.
|
|
74
|
+
eps : float, default=1e-8
|
|
75
|
+
Ridge-like regularisation added to the diagonal of the normal
|
|
76
|
+
equations to keep things numerically stable on rank-deficient
|
|
77
|
+
designs.
|
|
78
|
+
|
|
79
|
+
Attributes
|
|
80
|
+
----------
|
|
81
|
+
batch_levels_ : np.ndarray
|
|
82
|
+
Batch levels observed at ``fit`` time (in sorted order).
|
|
83
|
+
gamma_ : np.ndarray
|
|
84
|
+
Estimated batch coefficients in the sum-to-zero contrast basis,
|
|
85
|
+
of shape ``(n_contrasts, n_features)``.
|
|
86
|
+
contrasts_ : np.ndarray
|
|
87
|
+
Sum-to-zero contrast matrix used at fit time.
|
|
88
|
+
|
|
89
|
+
Notes
|
|
90
|
+
-----
|
|
91
|
+
* Unknown batch levels at ``transform`` raise a ``ValueError``. This
|
|
92
|
+
is intentional: Limma's subtraction is not defined for a batch that
|
|
93
|
+
was not part of the design.
|
|
94
|
+
* The design matrix is augmented with a constant column internally so
|
|
95
|
+
the intercept is absorbed into the OLS fit; only the batch
|
|
96
|
+
coefficients are subtracted on output.
|
|
97
|
+
|
|
98
|
+
References
|
|
99
|
+
----------
|
|
100
|
+
Ritchie, M.E. et al. (2015) "limma powers differential expression
|
|
101
|
+
analyses for RNA-sequencing and microarray studies."
|
|
102
|
+
Nucleic Acids Research 43(7): e47.
|
|
103
|
+
|
|
104
|
+
Examples
|
|
105
|
+
--------
|
|
106
|
+
>>> from maldibatchkit import Limma
|
|
107
|
+
>>> corrector = Limma(batch=batches, design=species_dummies)
|
|
108
|
+
>>> X_corrected = corrector.fit_transform(X)
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
def __init__(
|
|
112
|
+
self,
|
|
113
|
+
batch: ArrayLike,
|
|
114
|
+
*,
|
|
115
|
+
design: ArrayLike | None = None,
|
|
116
|
+
eps: float = 1e-8,
|
|
117
|
+
) -> None:
|
|
118
|
+
super().__init__(batch=batch)
|
|
119
|
+
self.design = design
|
|
120
|
+
self.eps = eps
|
|
121
|
+
|
|
122
|
+
def _design_at(self, idx: pd.Index) -> np.ndarray | None:
|
|
123
|
+
if self.design is None:
|
|
124
|
+
return None
|
|
125
|
+
sub = _subset(self.design, idx)
|
|
126
|
+
arr = (
|
|
127
|
+
sub.to_numpy(dtype=float)
|
|
128
|
+
if hasattr(sub, "to_numpy")
|
|
129
|
+
else np.asarray(sub, dtype=float)
|
|
130
|
+
)
|
|
131
|
+
if arr.ndim == 1:
|
|
132
|
+
arr = arr.reshape(-1, 1)
|
|
133
|
+
return arr
|
|
134
|
+
|
|
135
|
+
def _fit_impl(self, X_df: pd.DataFrame, batch: npt.NDArray[Any]) -> None:
|
|
136
|
+
levels = np.unique(batch)
|
|
137
|
+
self.batch_levels_ = levels
|
|
138
|
+
batch_design, contrasts = _design_matrix(batch, levels)
|
|
139
|
+
self.contrasts_ = contrasts
|
|
140
|
+
|
|
141
|
+
intercept = np.ones((X_df.shape[0], 1))
|
|
142
|
+
cov = self._design_at(X_df.index)
|
|
143
|
+
if cov is not None:
|
|
144
|
+
full_design = np.hstack([intercept, cov, batch_design])
|
|
145
|
+
n_before = 1 + cov.shape[1]
|
|
146
|
+
else:
|
|
147
|
+
full_design = np.hstack([intercept, batch_design])
|
|
148
|
+
n_before = 1
|
|
149
|
+
|
|
150
|
+
# Tikhonov-regularised normal equations
|
|
151
|
+
gram = full_design.T @ full_design
|
|
152
|
+
gram += self.eps * np.eye(gram.shape[0])
|
|
153
|
+
rhs = full_design.T @ X_df.to_numpy(dtype=float)
|
|
154
|
+
beta = la.solve(gram, rhs)
|
|
155
|
+
|
|
156
|
+
# Batch coefficients live in the last n_contrasts rows
|
|
157
|
+
self.gamma_ = beta[n_before:, :]
|
|
158
|
+
|
|
159
|
+
def _transform_impl(
|
|
160
|
+
self, X_df: pd.DataFrame, batch: npt.NDArray[Any]
|
|
161
|
+
) -> pd.DataFrame:
|
|
162
|
+
batch_design, _ = _design_matrix(batch, self.batch_levels_)
|
|
163
|
+
correction = batch_design @ self.gamma_
|
|
164
|
+
out = X_df.to_numpy(dtype=float) - correction
|
|
165
|
+
return pd.DataFrame(out, index=X_df.index, columns=X_df.columns)
|