MaldiBatchKit 0.2.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1/MaldiBatchKit.egg-info}/PKG-INFO +8 -4
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/MaldiBatchKit.egg-info/requires.txt +1 -1
- {maldibatchkit-0.2.0/MaldiBatchKit.egg-info → maldibatchkit-0.2.1}/PKG-INFO +8 -4
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/README.md +6 -2
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/__init__.py +1 -1
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/_base.py +40 -6
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/_cli_io.py +93 -60
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/_utils.py +79 -5
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/cli.py +69 -11
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/corrections/auto.py +56 -30
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/corrections/baselines.py +22 -13
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/corrections/combat.py +55 -12
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/corrections/harmony.py +58 -74
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/corrections/limma.py +4 -2
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/corrections/maldi.py +1 -1
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/corrections/quality_weighted.py +58 -19
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/diagnostics/benchmark.py +107 -34
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/diagnostics/generic.py +5 -4
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/diagnostics/maldi.py +9 -5
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/diagnostics/report.py +41 -20
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/integrations/maldiset.py +76 -28
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/metrics.py +27 -22
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/viz/peaks.py +2 -2
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/viz/summary.py +0 -3
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/viz/umap.py +4 -2
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/requirements.txt +1 -1
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/tests/test_cli.py +178 -2
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/tests/test_cli_io.py +46 -9
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/tests/test_metrics.py +55 -4
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/tests/test_mz_axis_resolution.py +16 -5
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/tests/test_no_data_leakage.py +1 -20
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/LICENSE +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/MaldiBatchKit.egg-info/SOURCES.txt +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/MaldiBatchKit.egg-info/dependency_links.txt +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/MaldiBatchKit.egg-info/entry_points.txt +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/MaldiBatchKit.egg-info/top_level.txt +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/corrections/__init__.py +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/corrections/noop.py +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/diagnostics/__init__.py +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/integrations/__init__.py +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/py.typed +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/maldibatchkit/viz/__init__.py +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/pyproject.toml +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/requirements-dev.txt +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/requirements-docs.txt +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/requirements-viz.txt +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/setup.cfg +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/tests/test_package.py +0 -0
- {maldibatchkit-0.2.0 → maldibatchkit-0.2.1}/tests/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: MaldiBatchKit
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Batch-effect correction methods for MALDI-TOF spectra in clinical AMR prediction workflows
|
|
5
5
|
Author-email: Ettore Rocchi <ettore.rocchi3@unibo.it>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -27,7 +27,7 @@ Requires-Dist: pandas>=1.3.0
|
|
|
27
27
|
Requires-Dist: scipy>=1.13.0
|
|
28
28
|
Requires-Dist: scikit-learn>=1.6.0
|
|
29
29
|
Requires-Dist: matplotlib>=3.4.0
|
|
30
|
-
Requires-Dist: combatlearn>=2.
|
|
30
|
+
Requires-Dist: combatlearn>=2.4.0
|
|
31
31
|
Requires-Dist: harmonypy<2,>=0.2.0
|
|
32
32
|
Requires-Dist: maldiamrkit>=0.15.0
|
|
33
33
|
Requires-Dist: typer>=0.9.0
|
|
@@ -69,6 +69,10 @@ Dynamic: license-file
|
|
|
69
69
|
<strong>Batch-effect correction methods for MALDI-TOF spectra in clinical AMR prediction workflows</strong>
|
|
70
70
|
</p>
|
|
71
71
|
|
|
72
|
+
<p align="center">
|
|
73
|
+
<img src="docs/batch_correction.svg" alt="Animated scatter plot: before correction MALDI-TOF samples cluster by acquisition site; after MaldiBatchKit the site effect is removed and samples cluster by species" width="720"/>
|
|
74
|
+
</p>
|
|
75
|
+
|
|
72
76
|
<p align="center">
|
|
73
77
|
<a href="#installation">Installation</a> •
|
|
74
78
|
<a href="#features">Features</a> •
|
|
@@ -212,7 +216,7 @@ maldibatchkit correct combat-fortin \
|
|
|
212
216
|
-o corrected.npz
|
|
213
217
|
```
|
|
214
218
|
|
|
215
|
-
Run `maldibatchkit correct <method> --help` for the full flag list of any corrector. `combat-fortin`
|
|
219
|
+
Run `maldibatchkit correct <method> --help` for the full flag list of any corrector. `combat-fortin` refuses to run without covariates (it would silently reduce to Johnson ComBat); `combat-chen` without covariates is plain CovBat (Johnson ComBat plus the covariance correction); `species-combat` / `quality-combat` require their dedicated `--species-csv` / `--quality-csv` inputs.
|
|
216
220
|
|
|
217
221
|
## Algorithms
|
|
218
222
|
|
|
@@ -220,7 +224,7 @@ Run `maldibatchkit correct <method> --help` for the full flag list of any correc
|
|
|
220
224
|
|--------------------------------|---------------------------|----------------------|------------------|
|
|
221
225
|
| ComBat (Johnson, Fortin, Chen) | `ComBat` | Fortin / Chen | yes |
|
|
222
226
|
| Limma | `Limma` | via `design=` | yes |
|
|
223
|
-
| Harmony | `Harmony` |
|
|
227
|
+
| Harmony | `Harmony` | no (removes them) | yes |
|
|
224
228
|
| Median centering | `MedianCentering` | no | yes |
|
|
225
229
|
| Z-score per batch | `ZScorePerBatch` | no | yes |
|
|
226
230
|
| Reference scaling | `ReferenceScaling` | no | yes |
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: MaldiBatchKit
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Batch-effect correction methods for MALDI-TOF spectra in clinical AMR prediction workflows
|
|
5
5
|
Author-email: Ettore Rocchi <ettore.rocchi3@unibo.it>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -27,7 +27,7 @@ Requires-Dist: pandas>=1.3.0
|
|
|
27
27
|
Requires-Dist: scipy>=1.13.0
|
|
28
28
|
Requires-Dist: scikit-learn>=1.6.0
|
|
29
29
|
Requires-Dist: matplotlib>=3.4.0
|
|
30
|
-
Requires-Dist: combatlearn>=2.
|
|
30
|
+
Requires-Dist: combatlearn>=2.4.0
|
|
31
31
|
Requires-Dist: harmonypy<2,>=0.2.0
|
|
32
32
|
Requires-Dist: maldiamrkit>=0.15.0
|
|
33
33
|
Requires-Dist: typer>=0.9.0
|
|
@@ -69,6 +69,10 @@ Dynamic: license-file
|
|
|
69
69
|
<strong>Batch-effect correction methods for MALDI-TOF spectra in clinical AMR prediction workflows</strong>
|
|
70
70
|
</p>
|
|
71
71
|
|
|
72
|
+
<p align="center">
|
|
73
|
+
<img src="docs/batch_correction.svg" alt="Animated scatter plot: before correction MALDI-TOF samples cluster by acquisition site; after MaldiBatchKit the site effect is removed and samples cluster by species" width="720"/>
|
|
74
|
+
</p>
|
|
75
|
+
|
|
72
76
|
<p align="center">
|
|
73
77
|
<a href="#installation">Installation</a> •
|
|
74
78
|
<a href="#features">Features</a> •
|
|
@@ -212,7 +216,7 @@ maldibatchkit correct combat-fortin \
|
|
|
212
216
|
-o corrected.npz
|
|
213
217
|
```
|
|
214
218
|
|
|
215
|
-
Run `maldibatchkit correct <method> --help` for the full flag list of any corrector. `combat-fortin`
|
|
219
|
+
Run `maldibatchkit correct <method> --help` for the full flag list of any corrector. `combat-fortin` refuses to run without covariates (it would silently reduce to Johnson ComBat); `combat-chen` without covariates is plain CovBat (Johnson ComBat plus the covariance correction); `species-combat` / `quality-combat` require their dedicated `--species-csv` / `--quality-csv` inputs.
|
|
216
220
|
|
|
217
221
|
## Algorithms
|
|
218
222
|
|
|
@@ -220,7 +224,7 @@ Run `maldibatchkit correct <method> --help` for the full flag list of any correc
|
|
|
220
224
|
|--------------------------------|---------------------------|----------------------|------------------|
|
|
221
225
|
| ComBat (Johnson, Fortin, Chen) | `ComBat` | Fortin / Chen | yes |
|
|
222
226
|
| Limma | `Limma` | via `design=` | yes |
|
|
223
|
-
| Harmony | `Harmony` |
|
|
227
|
+
| Harmony | `Harmony` | no (removes them) | yes |
|
|
224
228
|
| Median centering | `MedianCentering` | no | yes |
|
|
225
229
|
| Z-score per batch | `ZScorePerBatch` | no | yes |
|
|
226
230
|
| Reference scaling | `ReferenceScaling` | no | yes |
|
|
@@ -16,6 +16,10 @@
|
|
|
16
16
|
<strong>Batch-effect correction methods for MALDI-TOF spectra in clinical AMR prediction workflows</strong>
|
|
17
17
|
</p>
|
|
18
18
|
|
|
19
|
+
<p align="center">
|
|
20
|
+
<img src="docs/batch_correction.svg" alt="Animated scatter plot: before correction MALDI-TOF samples cluster by acquisition site; after MaldiBatchKit the site effect is removed and samples cluster by species" width="720"/>
|
|
21
|
+
</p>
|
|
22
|
+
|
|
19
23
|
<p align="center">
|
|
20
24
|
<a href="#installation">Installation</a> •
|
|
21
25
|
<a href="#features">Features</a> •
|
|
@@ -159,7 +163,7 @@ maldibatchkit correct combat-fortin \
|
|
|
159
163
|
-o corrected.npz
|
|
160
164
|
```
|
|
161
165
|
|
|
162
|
-
Run `maldibatchkit correct <method> --help` for the full flag list of any corrector. `combat-fortin`
|
|
166
|
+
Run `maldibatchkit correct <method> --help` for the full flag list of any corrector. `combat-fortin` refuses to run without covariates (it would silently reduce to Johnson ComBat); `combat-chen` without covariates is plain CovBat (Johnson ComBat plus the covariance correction); `species-combat` / `quality-combat` require their dedicated `--species-csv` / `--quality-csv` inputs.
|
|
163
167
|
|
|
164
168
|
## Algorithms
|
|
165
169
|
|
|
@@ -167,7 +171,7 @@ Run `maldibatchkit correct <method> --help` for the full flag list of any correc
|
|
|
167
171
|
|--------------------------------|---------------------------|----------------------|------------------|
|
|
168
172
|
| ComBat (Johnson, Fortin, Chen) | `ComBat` | Fortin / Chen | yes |
|
|
169
173
|
| Limma | `Limma` | via `design=` | yes |
|
|
170
|
-
| Harmony | `Harmony` |
|
|
174
|
+
| Harmony | `Harmony` | no (removes them) | yes |
|
|
171
175
|
| Median centering | `MedianCentering` | no | yes |
|
|
172
176
|
| Z-score per batch | `ZScorePerBatch` | no | yes |
|
|
173
177
|
| Reference scaling | `ReferenceScaling` | no | yes |
|
|
@@ -27,7 +27,14 @@ import pandas as pd
|
|
|
27
27
|
from sklearn.base import BaseEstimator, TransformerMixin
|
|
28
28
|
from sklearn.utils.validation import check_array, check_is_fitted
|
|
29
29
|
|
|
30
|
-
from ._utils import
|
|
30
|
+
from ._utils import (
|
|
31
|
+
ArrayLike,
|
|
32
|
+
_check_label_values,
|
|
33
|
+
_check_positional_alignment,
|
|
34
|
+
_check_unique_ids,
|
|
35
|
+
_ensure_index,
|
|
36
|
+
_subset,
|
|
37
|
+
)
|
|
31
38
|
|
|
32
39
|
|
|
33
40
|
def _check_array_finite(X: Any) -> None:
|
|
@@ -65,6 +72,8 @@ class BaseBatchCorrector(BaseEstimator, TransformerMixin): # type: ignore[misc]
|
|
|
65
72
|
:meth:`get_params` / :meth:`set_params` work.
|
|
66
73
|
"""
|
|
67
74
|
|
|
75
|
+
_sample_aligned_params: tuple[str, ...] = ("batch",)
|
|
76
|
+
|
|
68
77
|
def __init__(self, batch: ArrayLike) -> None:
|
|
69
78
|
self.batch = batch
|
|
70
79
|
|
|
@@ -72,6 +81,13 @@ class BaseBatchCorrector(BaseEstimator, TransformerMixin): # type: ignore[misc]
|
|
|
72
81
|
"""Validate the feature matrix."""
|
|
73
82
|
_check_array_finite(X)
|
|
74
83
|
|
|
84
|
+
def _check_sample_alignment(self, X: ArrayLike) -> None:
|
|
85
|
+
"""Reject pandas per-sample inputs that an ndarray ``X`` cannot match."""
|
|
86
|
+
if isinstance(X, pd.DataFrame | pd.Series):
|
|
87
|
+
return
|
|
88
|
+
for name in self._sample_aligned_params:
|
|
89
|
+
_check_positional_alignment(getattr(self, name, None), len(X), name)
|
|
90
|
+
|
|
75
91
|
def _prepare_fit(
|
|
76
92
|
self, X: ArrayLike
|
|
77
93
|
) -> tuple[pd.DataFrame, npt.NDArray[Any], pd.Index]:
|
|
@@ -87,7 +103,10 @@ class BaseBatchCorrector(BaseEstimator, TransformerMixin): # type: ignore[misc]
|
|
|
87
103
|
The shared index used by ``X`` and ``batch``.
|
|
88
104
|
"""
|
|
89
105
|
self._validate_X(X)
|
|
106
|
+
self._check_sample_alignment(X)
|
|
90
107
|
idx = _ensure_index(X)
|
|
108
|
+
_check_unique_ids(idx, "X")
|
|
109
|
+
self._fitted_on_frame_ = isinstance(X, pd.DataFrame)
|
|
91
110
|
if isinstance(X, pd.DataFrame):
|
|
92
111
|
self.feature_names_in_ = np.asarray(X.columns, dtype=object)
|
|
93
112
|
X_df = X
|
|
@@ -104,11 +123,7 @@ class BaseBatchCorrector(BaseEstimator, TransformerMixin): # type: ignore[misc]
|
|
|
104
123
|
raise ValueError(f"`batch` must be 1-D; got shape {batch_vec.shape}.")
|
|
105
124
|
batch_vec = batch_vec.iloc[:, 0]
|
|
106
125
|
batch_ser = pd.Series(batch_vec, index=idx)
|
|
107
|
-
|
|
108
|
-
raise ValueError(
|
|
109
|
-
f"batch contains {int(batch_ser.isna().sum())} NaN value(s); "
|
|
110
|
-
f"all batch labels must be non-null."
|
|
111
|
-
)
|
|
126
|
+
_check_label_values(batch_ser, "batch")
|
|
112
127
|
return X_df, batch_ser.to_numpy(), idx
|
|
113
128
|
|
|
114
129
|
def _prepare_transform(
|
|
@@ -117,13 +132,32 @@ class BaseBatchCorrector(BaseEstimator, TransformerMixin): # type: ignore[misc]
|
|
|
117
132
|
"""Run common ``transform``-time bookkeeping."""
|
|
118
133
|
check_is_fitted(self)
|
|
119
134
|
self._validate_X(X)
|
|
135
|
+
self._check_sample_alignment(X)
|
|
120
136
|
idx = _ensure_index(X)
|
|
137
|
+
_check_unique_ids(idx, "X")
|
|
121
138
|
was_df = isinstance(X, pd.DataFrame)
|
|
122
139
|
X_df = X if was_df else pd.DataFrame(np.asarray(X), index=idx)
|
|
140
|
+
if X_df.shape[1] != self.n_features_in_:
|
|
141
|
+
raise ValueError(
|
|
142
|
+
f"X has {X_df.shape[1]} features, but {type(self).__name__} "
|
|
143
|
+
f"is expecting {self.n_features_in_} features as input."
|
|
144
|
+
)
|
|
145
|
+
if (
|
|
146
|
+
was_df
|
|
147
|
+
and getattr(self, "_fitted_on_frame_", False)
|
|
148
|
+
and not np.array_equal(
|
|
149
|
+
np.asarray(X.columns, dtype=object), self.feature_names_in_
|
|
150
|
+
)
|
|
151
|
+
):
|
|
152
|
+
raise ValueError(
|
|
153
|
+
"The feature names should match those that were passed during "
|
|
154
|
+
"fit, in the same order."
|
|
155
|
+
)
|
|
123
156
|
batch_vec = _subset(self.batch, idx)
|
|
124
157
|
if isinstance(batch_vec, pd.DataFrame):
|
|
125
158
|
batch_vec = batch_vec.iloc[:, 0]
|
|
126
159
|
batch_ser = pd.Series(batch_vec, index=idx)
|
|
160
|
+
_check_label_values(batch_ser, "batch")
|
|
127
161
|
return X_df, batch_ser.to_numpy(), idx, was_df
|
|
128
162
|
|
|
129
163
|
def _fit_impl(
|
|
@@ -13,20 +13,25 @@ Supported input / output containers
|
|
|
13
13
|
``columns``, ``index``, ``batch``, and arbitrary extra arrays kept as
|
|
14
14
|
passthrough metadata. This is the MaldiSet-friendly format.
|
|
15
15
|
|
|
16
|
-
All auxiliary loaders align the loaded object to ``X.index``
|
|
17
|
-
|
|
18
|
-
|
|
16
|
+
All auxiliary loaders align the loaded object to ``X.index`` by sample
|
|
17
|
+
ID and fail, listing the missing IDs, when the sidecar does not cover
|
|
18
|
+
``X.index``. Sample IDs and labels are read as strings (``"001"`` stays
|
|
19
|
+
``"001"`` and ``"NA"`` is a label, not a missing value).
|
|
19
20
|
"""
|
|
20
21
|
|
|
21
22
|
from __future__ import annotations
|
|
22
23
|
|
|
23
24
|
from pathlib import Path
|
|
25
|
+
from typing import Any
|
|
24
26
|
|
|
25
27
|
import numpy as np
|
|
26
28
|
import pandas as pd
|
|
27
29
|
import typer
|
|
28
30
|
|
|
31
|
+
from ._utils import _check_unique_ids
|
|
32
|
+
|
|
29
33
|
__all__ = [
|
|
34
|
+
"load_features",
|
|
30
35
|
"load_matrix",
|
|
31
36
|
"load_dataframe_csv",
|
|
32
37
|
"load_labels_csv",
|
|
@@ -35,9 +40,41 @@ __all__ = [
|
|
|
35
40
|
]
|
|
36
41
|
|
|
37
42
|
|
|
38
|
-
def _read_csv_indexed(path: Path) -> pd.DataFrame:
|
|
39
|
-
"""Read a CSV
|
|
40
|
-
|
|
43
|
+
def _read_csv_indexed(path: Path, **kwargs: Any) -> pd.DataFrame:
|
|
44
|
+
"""Read a CSV whose first column holds the sample IDs, kept as strings."""
|
|
45
|
+
df = pd.read_csv(path, index_col=0, **kwargs)
|
|
46
|
+
ids = pd.read_csv(path, usecols=[0], dtype=str, keep_default_na=False)
|
|
47
|
+
df.index = pd.Index(ids.iloc[:, 0].to_numpy(), name=df.index.name)
|
|
48
|
+
return df
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _load_npz(path: Path) -> tuple[pd.DataFrame, pd.Series | None, dict]:
|
|
52
|
+
"""Return ``X``, the bundled batch (or None) and the extra arrays of an NPZ."""
|
|
53
|
+
with np.load(path, allow_pickle=True) as npz:
|
|
54
|
+
if "X" not in npz.files:
|
|
55
|
+
raise typer.BadParameter(
|
|
56
|
+
f"{path} is missing the 'X' array. Expected "
|
|
57
|
+
f"np.savez(..., X=, [columns=, index=, batch=])."
|
|
58
|
+
)
|
|
59
|
+
X_arr = np.asarray(npz["X"])
|
|
60
|
+
columns = list(npz["columns"]) if "columns" in npz.files else None
|
|
61
|
+
index = list(npz["index"]) if "index" in npz.files else None
|
|
62
|
+
npz_batch = (
|
|
63
|
+
pd.Series(np.asarray(npz["batch"])) if "batch" in npz.files else None
|
|
64
|
+
)
|
|
65
|
+
extras = {
|
|
66
|
+
k: np.asarray(npz[k])
|
|
67
|
+
for k in npz.files
|
|
68
|
+
if k not in {"X", "columns", "index", "batch"}
|
|
69
|
+
}
|
|
70
|
+
return pd.DataFrame(X_arr, index=index, columns=columns), npz_batch, extras
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def load_features(path: Path) -> pd.DataFrame:
|
|
74
|
+
"""Load only the feature matrix of a CSV or NPZ file."""
|
|
75
|
+
if path.suffix.lower() == ".npz":
|
|
76
|
+
return _load_npz(path)[0]
|
|
77
|
+
return _read_csv_indexed(path)
|
|
41
78
|
|
|
42
79
|
|
|
43
80
|
def load_matrix(
|
|
@@ -65,24 +102,7 @@ def load_matrix(
|
|
|
65
102
|
columns / batch). Empty for CSV inputs.
|
|
66
103
|
"""
|
|
67
104
|
if path.suffix.lower() == ".npz":
|
|
68
|
-
|
|
69
|
-
if "X" not in npz.files:
|
|
70
|
-
raise typer.BadParameter(
|
|
71
|
-
f"{path} is missing the 'X' array. Expected "
|
|
72
|
-
f"np.savez(..., X=, [columns=, index=, batch=])."
|
|
73
|
-
)
|
|
74
|
-
X_arr = np.asarray(npz["X"])
|
|
75
|
-
columns = list(npz["columns"]) if "columns" in npz.files else None
|
|
76
|
-
index = list(npz["index"]) if "index" in npz.files else None
|
|
77
|
-
npz_batch = (
|
|
78
|
-
pd.Series(np.asarray(npz["batch"])) if "batch" in npz.files else None
|
|
79
|
-
)
|
|
80
|
-
extras = {
|
|
81
|
-
k: np.asarray(npz[k])
|
|
82
|
-
for k in npz.files
|
|
83
|
-
if k not in {"X", "columns", "index", "batch"}
|
|
84
|
-
}
|
|
85
|
-
X = pd.DataFrame(X_arr, index=index, columns=columns)
|
|
105
|
+
X, npz_batch, extras = _load_npz(path)
|
|
86
106
|
if npz_batch is not None:
|
|
87
107
|
if len(npz_batch) != len(X):
|
|
88
108
|
raise typer.BadParameter(
|
|
@@ -98,7 +118,6 @@ def load_matrix(
|
|
|
98
118
|
batch = load_labels_csv(batch_csv, X.index)
|
|
99
119
|
return X, batch, extras
|
|
100
120
|
|
|
101
|
-
# CSV path
|
|
102
121
|
X = _read_csv_indexed(path)
|
|
103
122
|
if batch_csv is None:
|
|
104
123
|
raise typer.BadParameter(
|
|
@@ -112,15 +131,11 @@ def load_matrix(
|
|
|
112
131
|
def load_labels_csv(path: Path, index: pd.Index) -> pd.Series:
|
|
113
132
|
"""Load a single-column sidecar CSV as a Series aligned to ``index``.
|
|
114
133
|
|
|
115
|
-
The CSV's first column is
|
|
116
|
-
|
|
117
|
-
index (i.e. a single data column read with the default
|
|
118
|
-
``RangeIndex``), the labels are assumed to already be in ``X`` order
|
|
119
|
-
and are aligned positionally.
|
|
134
|
+
The CSV's first column is the sample ID and the second the label, both
|
|
135
|
+
read as strings.
|
|
120
136
|
"""
|
|
121
|
-
df =
|
|
137
|
+
df = _read_csv_indexed(path, dtype=str, keep_default_na=False)
|
|
122
138
|
if df.shape[1] == 0:
|
|
123
|
-
# Only an index column was present -> data is the index itself
|
|
124
139
|
raise typer.BadParameter(
|
|
125
140
|
f"{path} has no data columns. Expected <sample_id>,<label>."
|
|
126
141
|
)
|
|
@@ -134,9 +149,26 @@ def load_labels_csv(path: Path, index: pd.Index) -> pd.Series:
|
|
|
134
149
|
return _align_series(series, index, source=path)
|
|
135
150
|
|
|
136
151
|
|
|
137
|
-
def load_dataframe_csv(
|
|
138
|
-
|
|
139
|
-
|
|
152
|
+
def load_dataframe_csv(
|
|
153
|
+
path: Path, index: pd.Index, *, categorical: bool = False
|
|
154
|
+
) -> pd.DataFrame:
|
|
155
|
+
"""Load a multi-column sidecar CSV and align it to ``index``.
|
|
156
|
+
|
|
157
|
+
Parameters
|
|
158
|
+
----------
|
|
159
|
+
path : Path
|
|
160
|
+
CSV whose first column holds the sample IDs.
|
|
161
|
+
index : pd.Index
|
|
162
|
+
``X.index`` to align to.
|
|
163
|
+
categorical : bool, default=False
|
|
164
|
+
Read the values as strings (labels such as ``"NA"`` or ``"001"``
|
|
165
|
+
are kept as written); otherwise they are parsed as numbers.
|
|
166
|
+
"""
|
|
167
|
+
df = (
|
|
168
|
+
_read_csv_indexed(path, dtype=str, keep_default_na=False)
|
|
169
|
+
if categorical
|
|
170
|
+
else _read_csv_indexed(path)
|
|
171
|
+
)
|
|
140
172
|
if df.shape[1] == 0:
|
|
141
173
|
raise typer.BadParameter(f"{path} has no data columns.")
|
|
142
174
|
return _align_frame(df, index, source=path)
|
|
@@ -180,32 +212,33 @@ def save_matrix(
|
|
|
180
212
|
X.to_csv(path)
|
|
181
213
|
|
|
182
214
|
|
|
215
|
+
def _align_by_id(obj, index: pd.Index, *, source: Path):
|
|
216
|
+
"""Reindex ``obj`` onto ``index`` by sample ID (compared as strings)."""
|
|
217
|
+
if obj.index.equals(index):
|
|
218
|
+
return obj
|
|
219
|
+
keys = pd.Index(index).astype(str)
|
|
220
|
+
if obj.index.name is not None and str(obj.index.name) in keys:
|
|
221
|
+
raise typer.BadParameter(
|
|
222
|
+
f"{source} has no header row: its first line starts with sample ID "
|
|
223
|
+
f"{obj.index.name!r}. Add a header such as 'id,label'."
|
|
224
|
+
)
|
|
225
|
+
try:
|
|
226
|
+
_check_unique_ids(obj.index, str(source))
|
|
227
|
+
except ValueError as exc:
|
|
228
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
229
|
+
by_id = obj.set_axis(obj.index.astype(str), axis=0)
|
|
230
|
+
missing = keys[~keys.isin(by_id.index)]
|
|
231
|
+
if len(missing):
|
|
232
|
+
raise typer.BadParameter(
|
|
233
|
+
f"{source}: cannot align to X.index; {len(missing)} sample ID(s) "
|
|
234
|
+
f"of X are missing (first 5: {list(missing[:5])})."
|
|
235
|
+
)
|
|
236
|
+
return by_id.reindex(keys).set_axis(index, axis=0)
|
|
237
|
+
|
|
238
|
+
|
|
183
239
|
def _align_series(series: pd.Series, index: pd.Index, *, source: Path) -> pd.Series:
|
|
184
|
-
|
|
185
|
-
return series
|
|
186
|
-
if set(index).issubset(set(series.index)):
|
|
187
|
-
return series.reindex(index)
|
|
188
|
-
if len(series) == len(index):
|
|
189
|
-
# Positional fallback: the CSV did not carry the X sample IDs.
|
|
190
|
-
aligned = series.copy()
|
|
191
|
-
aligned.index = index
|
|
192
|
-
return aligned
|
|
193
|
-
raise typer.BadParameter(
|
|
194
|
-
f"{source}: cannot align labels to X.index. "
|
|
195
|
-
f"Labels have {len(series)} rows, X has {len(index)} rows, "
|
|
196
|
-
f"and label indices do not cover X.index."
|
|
197
|
-
)
|
|
240
|
+
return _align_by_id(series, index, source=source)
|
|
198
241
|
|
|
199
242
|
|
|
200
243
|
def _align_frame(df: pd.DataFrame, index: pd.Index, *, source: Path) -> pd.DataFrame:
|
|
201
|
-
|
|
202
|
-
return df
|
|
203
|
-
if set(index).issubset(set(df.index)):
|
|
204
|
-
return df.reindex(index)
|
|
205
|
-
if len(df) == len(index):
|
|
206
|
-
aligned = df.copy()
|
|
207
|
-
aligned.index = index
|
|
208
|
-
return aligned
|
|
209
|
-
raise typer.BadParameter(
|
|
210
|
-
f"{source}: cannot align to X.index. Got {len(df)} rows, X has {len(index)}."
|
|
211
|
-
)
|
|
244
|
+
return _align_by_id(df, index, source=source)
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import warnings
|
|
5
6
|
from typing import Any
|
|
6
7
|
|
|
7
8
|
import numpy as np
|
|
@@ -45,6 +46,7 @@ def _subset(obj: ArrayLike | None, idx: pd.Index) -> pd.Series | pd.DataFrame |
|
|
|
45
46
|
if obj is None:
|
|
46
47
|
return None
|
|
47
48
|
if isinstance(obj, pd.Series | pd.DataFrame):
|
|
49
|
+
_check_unique_ids(obj.index, "batch/covariate index")
|
|
48
50
|
try:
|
|
49
51
|
return obj.loc[idx]
|
|
50
52
|
except KeyError as exc:
|
|
@@ -62,6 +64,79 @@ def _subset(obj: ArrayLike | None, idx: pd.Index) -> pd.Series | pd.DataFrame |
|
|
|
62
64
|
return pd.DataFrame(arr, index=idx)
|
|
63
65
|
|
|
64
66
|
|
|
67
|
+
def _check_positional_alignment(obj: ArrayLike | None, n_rows: int, name: str) -> None:
|
|
68
|
+
"""Raise if pandas ``obj`` is not indexed by ``RangeIndex(n_rows)``."""
|
|
69
|
+
if isinstance(obj, pd.Series | pd.DataFrame) and not obj.index.equals(
|
|
70
|
+
pd.RangeIndex(n_rows)
|
|
71
|
+
):
|
|
72
|
+
raise ValueError(
|
|
73
|
+
f"`{name}` is a pandas object but X is an ndarray with no index, "
|
|
74
|
+
f"and `{name}`'s index is not RangeIndex({n_rows}), so its rows "
|
|
75
|
+
f"cannot be matched to X. Pass X as a DataFrame (in a Pipeline, "
|
|
76
|
+
f'call `set_output(transform="pandas")` on the previous steps) or '
|
|
77
|
+
f"pass `{name}` as a numpy array in the same row order as X."
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _warn_unscaled(n_pairs: int, eps: float) -> None:
|
|
82
|
+
"""Warn that ``n_pairs`` (batch, feature) scales <= ``eps`` are left unscaled."""
|
|
83
|
+
if n_pairs:
|
|
84
|
+
warnings.warn(
|
|
85
|
+
f"{n_pairs} (batch, feature) pair(s) have a training scale <= "
|
|
86
|
+
f"eps={eps:g} (e.g. a bin that is all zero in a batch); they are "
|
|
87
|
+
f"left unscaled.",
|
|
88
|
+
stacklevel=4,
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _check_labels(labels: ArrayLike, n_rows: int, name: str) -> npt.NDArray[Any]:
|
|
93
|
+
"""Return ``labels`` as a 1-D array of ``n_rows`` non-missing values or raise."""
|
|
94
|
+
arr = np.asarray(labels)
|
|
95
|
+
if arr.ndim != 1 or arr.shape[0] != n_rows:
|
|
96
|
+
raise ValueError(
|
|
97
|
+
f"`{name}` must hold one label per row of X ({n_rows}); got shape "
|
|
98
|
+
f"{arr.shape}."
|
|
99
|
+
)
|
|
100
|
+
_check_label_values(arr, name)
|
|
101
|
+
return arr
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _check_label_values(values: ArrayLike, name: str) -> None:
|
|
105
|
+
"""Raise on missing labels or labels mixing strings and non-strings."""
|
|
106
|
+
arr = np.asarray(values, dtype=object)
|
|
107
|
+
n_missing = int(pd.isna(arr).sum())
|
|
108
|
+
if n_missing:
|
|
109
|
+
raise ValueError(f"`{name}` contains {n_missing} missing (NaN) label(s).")
|
|
110
|
+
is_str = np.array([isinstance(v, str) for v in arr])
|
|
111
|
+
if is_str.any() and not is_str.all():
|
|
112
|
+
raise ValueError(
|
|
113
|
+
f"`{name}` mixes string and non-string labels (e.g. 1 and '1'); "
|
|
114
|
+
f"use one type for all labels."
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _check_unique_ids(index: pd.Index, what: str) -> None:
|
|
119
|
+
"""Raise if ``index`` holds duplicated sample IDs."""
|
|
120
|
+
if index.has_duplicates:
|
|
121
|
+
dup = index[index.duplicated()].unique().tolist()
|
|
122
|
+
raise ValueError(
|
|
123
|
+
f"{what} has duplicated sample ID(s) (first 3: {dup[:3]}); sample "
|
|
124
|
+
f"IDs must be unique."
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _check_min_batch_size(batch: ArrayLike) -> None:
|
|
129
|
+
"""Raise if any batch has fewer than 2 samples."""
|
|
130
|
+
counts = pd.Series(np.asarray(batch)).value_counts()
|
|
131
|
+
small = counts[counts < 2]
|
|
132
|
+
if len(small):
|
|
133
|
+
raise ValueError(
|
|
134
|
+
f"Batch {small.index[0]!r} has only {int(small.iloc[0])} sample(s). "
|
|
135
|
+
f"ComBat requires >= 2 samples per batch. Consider merging small "
|
|
136
|
+
f"batches or removing them."
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
|
|
65
140
|
def _ensure_index(X: ArrayLike) -> pd.Index:
|
|
66
141
|
"""Return ``X.index`` if ``X`` is a pandas object, else a default RangeIndex."""
|
|
67
142
|
if isinstance(X, pd.DataFrame | pd.Series):
|
|
@@ -99,13 +174,15 @@ def _resolve_mz_axis(
|
|
|
99
174
|
n_features = df.shape[1]
|
|
100
175
|
if mz_values is not None:
|
|
101
176
|
mz = np.asarray(mz_values, dtype=float)
|
|
177
|
+
if mz.ndim != 1:
|
|
178
|
+
raise ValueError(f"mz_values must be 1-D; got shape {mz.shape}.")
|
|
179
|
+
if not np.isfinite(mz).all():
|
|
180
|
+
raise ValueError("mz_values must be finite (no NaN or inf).")
|
|
102
181
|
if mz.shape[0] != n_features:
|
|
103
182
|
raise ValueError(
|
|
104
183
|
f"mz_values has length {mz.shape[0]} but X has {n_features} columns."
|
|
105
184
|
)
|
|
106
185
|
return mz, True
|
|
107
|
-
# pandas.DataFrame(ndarray) yields a default RangeIndex(0, n) -
|
|
108
|
-
# that is positional, not an m/z axis. Treat it as bin indices.
|
|
109
186
|
if isinstance(df.columns, pd.RangeIndex):
|
|
110
187
|
return np.arange(n_features, dtype=float), False
|
|
111
188
|
try:
|
|
@@ -114,9 +191,6 @@ def _resolve_mz_axis(
|
|
|
114
191
|
return np.arange(n_features, dtype=float), False
|
|
115
192
|
if np.any(np.isnan(mz)):
|
|
116
193
|
return np.arange(n_features, dtype=float), False
|
|
117
|
-
# If the coerced axis is exactly ``[0, 1, ..., n-1]`` it is an
|
|
118
|
-
# integer-labelled DataFrame that happens to parse as floats;
|
|
119
|
-
# the caller almost certainly did not mean those as m/z.
|
|
120
194
|
if np.array_equal(mz, np.arange(n_features, dtype=float)):
|
|
121
195
|
return mz, False
|
|
122
196
|
return mz, True
|