clone2vec 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- clone2vec/__init__.py +23 -0
- clone2vec/associations.py +1117 -0
- clone2vec/catboost.py +622 -0
- clone2vec/datasets.py +94 -0
- clone2vec/eigenvalues.py +187 -0
- clone2vec/embeddings.py +1204 -0
- clone2vec/integration.py +511 -0
- clone2vec/plotting.py +3540 -0
- clone2vec/preprocessing.py +660 -0
- clone2vec/seurat.py +720 -0
- clone2vec/tools.py +633 -0
- clone2vec/utils.py +618 -0
- clone2vec-0.1.0.dist-info/METADATA +86 -0
- clone2vec-0.1.0.dist-info/RECORD +16 -0
- clone2vec-0.1.0.dist-info/WHEEL +5 -0
- clone2vec-0.1.0.dist-info/top_level.txt +1 -0
clone2vec/catboost.py
ADDED
|
@@ -0,0 +1,622 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import scanpy as sc
|
|
4
|
+
import numpy as np
|
|
5
|
+
import scipy.sparse as sp
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
from typing import Literal
|
|
12
|
+
from tqdm import tqdm
|
|
13
|
+
|
|
14
|
+
from .utils import _nan_mask
|
|
15
|
+
from .associations import _clr, _fast_corr_cols
|
|
16
|
+
|
|
17
|
+
logg = sc.logging
|
|
18
|
+
|
|
19
|
+
class _catboost_tqdm:
|
|
20
|
+
"""
|
|
21
|
+
Custom progress bar for CatBoost training.
|
|
22
|
+
|
|
23
|
+
Parameters
|
|
24
|
+
----------
|
|
25
|
+
total_iterations : int
|
|
26
|
+
Total number of iterations.
|
|
27
|
+
show : bool, optional
|
|
28
|
+
Whether to show the progress bar, by default True.
|
|
29
|
+
prefix : str, optional
|
|
30
|
+
Prefix for the progress bar, by default "".
|
|
31
|
+
"""
|
|
32
|
+
def __init__(self, total_iterations, show: bool = True, prefix: str = ""):
|
|
33
|
+
if show:
|
|
34
|
+
self.pbar = tqdm(
|
|
35
|
+
total=total_iterations,
|
|
36
|
+
desc=f"{prefix}training",
|
|
37
|
+
file=sys.stdout,
|
|
38
|
+
)
|
|
39
|
+
else:
|
|
40
|
+
self.pbar = None
|
|
41
|
+
def write(self, message: str):
|
|
42
|
+
if self.pbar:
|
|
43
|
+
message_stripped = message.strip()
|
|
44
|
+
if message_stripped and message_stripped[0].isdigit() and ":" in message_stripped:
|
|
45
|
+
self.pbar.update(1)
|
|
46
|
+
try:
|
|
47
|
+
parts = message_stripped.split("\t")
|
|
48
|
+
self.pbar.set_postfix({
|
|
49
|
+
"train": float(parts[1].split(":")[-1]),
|
|
50
|
+
"validation": float(parts[2].split(":")[-1]),
|
|
51
|
+
})
|
|
52
|
+
except Exception:
|
|
53
|
+
pass
|
|
54
|
+
def flush(self):
|
|
55
|
+
if self.pbar:
|
|
56
|
+
sys.stdout.flush()
|
|
57
|
+
def close(self):
|
|
58
|
+
if self.pbar:
|
|
59
|
+
self.pbar.close()
|
|
60
|
+
def __enter__(self):
|
|
61
|
+
return self
|
|
62
|
+
def __exit__(self, exc_type, exc, tb):
|
|
63
|
+
self.close()
|
|
64
|
+
self.pbar = None
|
|
65
|
+
|
|
66
|
+
def _prepare_pools(
|
|
67
|
+
adata,
|
|
68
|
+
obsm_key: str,
|
|
69
|
+
gs_key: str | None = None,
|
|
70
|
+
validation_key: str | None = None,
|
|
71
|
+
validation_value: str | None = None,
|
|
72
|
+
features: list[str] | None = None,
|
|
73
|
+
use_raw: bool | None = None,
|
|
74
|
+
layer: str | None = None,
|
|
75
|
+
response_transform: None | Literal["logit", "log1p", "sqrt", "clr"] = None,
|
|
76
|
+
clr_pseudocount: float = 1e-3,
|
|
77
|
+
split_multi: bool = False,
|
|
78
|
+
min_size: int = 3,
|
|
79
|
+
loss: str = "MultiRMSE",
|
|
80
|
+
) -> tuple["Pool", "Pool", "Pool", np.ndarray]:
|
|
81
|
+
"""
|
|
82
|
+
Prepares CatBoost Pools with full, train, and validation datasets.
|
|
83
|
+
|
|
84
|
+
Parameters
|
|
85
|
+
----------
|
|
86
|
+
adata : AnnData
|
|
87
|
+
AnnData object with cell metadata and embeddings.
|
|
88
|
+
obsm_key : str
|
|
89
|
+
Key in `adata.obsm` to use as the embedding.
|
|
90
|
+
gs_key : str | None, optional
|
|
91
|
+
Key in `adata.uns` to use for group splitting. Default is None.
|
|
92
|
+
validation_key : str | None, optional
|
|
93
|
+
Column in `adata.obs` to use for validation split. Default is None.
|
|
94
|
+
validation_value : str | None, optional
|
|
95
|
+
Value in `validation_key` to use for validation split. Default is None.
|
|
96
|
+
features : list[str] | None, optional
|
|
97
|
+
List of features to use as predictors. Default is None.
|
|
98
|
+
use_raw : bool | None, optional
|
|
99
|
+
Whether to use `adata.raw` for feature selection. Default is None.
|
|
100
|
+
layer : str | None, optional
|
|
101
|
+
Layer in `adata.layers` to use for feature selection. Default is None.
|
|
102
|
+
response_transform : None | Literal["logit", "log1p", "sqrt"], optional
|
|
103
|
+
Transform to apply to the response variable. Default is None.
|
|
104
|
+
clr_pseudocount : float, optional
|
|
105
|
+
Pseudocount to add to expression values before CLR transformation, by default 1e-3.
|
|
106
|
+
split_multi : bool, optional
|
|
107
|
+
Whether to split multi-dimensional response into multiple pools. Default is False.
|
|
108
|
+
min_size : int, optional
|
|
109
|
+
Minimum amount of non-zero values for each feature in the train and validation datasets. Default is 3.
|
|
110
|
+
|
|
111
|
+
Returns
|
|
112
|
+
-------
|
|
113
|
+
tuple[Pool, Pool, Pool, np.ndarray]
|
|
114
|
+
Full, train, validation pools and validation mask.
|
|
115
|
+
"""
|
|
116
|
+
try:
|
|
117
|
+
from catboost import Pool
|
|
118
|
+
except ImportError:
|
|
119
|
+
raise ImportError("catboost package is not installed. Please install it using `pip install catboost`.")
|
|
120
|
+
|
|
121
|
+
if validation_key is None or validation_value is None:
|
|
122
|
+
if gs_key is None:
|
|
123
|
+
try:
|
|
124
|
+
validation_key = adata.uns["gs"]["obs_key"]
|
|
125
|
+
validation_value = adata.uns["gs"]["keys"]["validation"]
|
|
126
|
+
except:
|
|
127
|
+
raise ValueError("gs_key not found in adata.uns. Please run c2v.tl.gs() or set validation_key and validation_value.")
|
|
128
|
+
else:
|
|
129
|
+
validation_key = adata.uns[gs_key]["obs_key"]
|
|
130
|
+
validation_value = adata.uns[gs_key]["keys"]["validation"]
|
|
131
|
+
validation_mask = adata.obs[validation_key] == validation_value
|
|
132
|
+
train_mask = ~validation_mask
|
|
133
|
+
|
|
134
|
+
validation_mask = np.array(validation_mask)
|
|
135
|
+
train_mask = np.array(train_mask)
|
|
136
|
+
|
|
137
|
+
if obsm_key not in adata.obsm.keys():
|
|
138
|
+
raise ValueError(f"Key {obsm_key} not found in adata_train.obsm. Please provide a valid key.")
|
|
139
|
+
|
|
140
|
+
if adata.obsm[obsm_key].shape[1] > 1:
|
|
141
|
+
label = adata.obsm[obsm_key]
|
|
142
|
+
else:
|
|
143
|
+
label = adata.obsm[obsm_key].flatten()
|
|
144
|
+
|
|
145
|
+
if isinstance(label, pd.DataFrame):
|
|
146
|
+
label_names = label.columns
|
|
147
|
+
label = label.values
|
|
148
|
+
else:
|
|
149
|
+
label_names = np.arange(label.shape[1]).astype(str)
|
|
150
|
+
|
|
151
|
+
if split_multi:
|
|
152
|
+
# Checking if the label matrix is integer values
|
|
153
|
+
if not np.allclose(label, label.astype(int)) and ((np.round(label.sum(axis=1), 5) > 1).sum() > 0):
|
|
154
|
+
logg.warning("When using MultiClass loss, values must be probabilities or integers. Changing to MultiCrossEntropy.")
|
|
155
|
+
split_multi = False
|
|
156
|
+
loss = "MultiCrossEntropy"
|
|
157
|
+
|
|
158
|
+
if response_transform == "logit":
|
|
159
|
+
label = np.clip(label, 0.01, 0.99).copy()
|
|
160
|
+
label = np.log(label / (1 - label))
|
|
161
|
+
elif response_transform == "log1p":
|
|
162
|
+
label = np.log1p(label).copy()
|
|
163
|
+
elif response_transform == "sqrt":
|
|
164
|
+
label = np.sqrt(label).copy()
|
|
165
|
+
elif response_transform == "clr":
|
|
166
|
+
label = _clr(label, epsilon=clr_pseudocount).copy()
|
|
167
|
+
elif response_transform:
|
|
168
|
+
logg.warning("response_transform must be None, 'logit', 'log1p', 'sqrt', or 'clr'. Setting response_transform=None.")
|
|
169
|
+
response_transform = None
|
|
170
|
+
|
|
171
|
+
for feature in features:
|
|
172
|
+
if feature not in adata.obs.columns:
|
|
173
|
+
raise ValueError(f"Feature {feature} not found in adata_train.obs. Please provide a valid feature.")
|
|
174
|
+
|
|
175
|
+
cat_features = []
|
|
176
|
+
for feature in features:
|
|
177
|
+
if isinstance(adata.obs[feature], pd.CategoricalDtype) or \
|
|
178
|
+
isinstance(adata.obs[feature], pd.StringDtype) or \
|
|
179
|
+
pd.api.types.is_object_dtype(adata.obs[feature]):
|
|
180
|
+
cat_features.append(feature)
|
|
181
|
+
|
|
182
|
+
if layer:
|
|
183
|
+
X = adata.layers[layer]
|
|
184
|
+
var_names = adata.var_names
|
|
185
|
+
elif use_raw:
|
|
186
|
+
X = adata.raw.X
|
|
187
|
+
var_names = adata.raw.var_names
|
|
188
|
+
else:
|
|
189
|
+
X = adata.X
|
|
190
|
+
var_names = adata.var_names
|
|
191
|
+
|
|
192
|
+
fate_size_train = (label[train_mask] != 0).sum(axis=0)
|
|
193
|
+
fate_size_validation = (label[validation_mask] != 0).sum(axis=0)
|
|
194
|
+
|
|
195
|
+
fates_mask_train = fate_size_train >= min_size
|
|
196
|
+
fates_mask_validation = fate_size_validation >= min_size
|
|
197
|
+
fates_mask = fates_mask_train & fates_mask_validation
|
|
198
|
+
|
|
199
|
+
if sum(~fates_mask) > 0:
|
|
200
|
+
logg.warning(f"Removing fates with less than {min_size} clones in validation and/or train from the analysis ({', '.join(label_names[~fates_mask])}).")
|
|
201
|
+
|
|
202
|
+
if sp.issparse(adata.X):
|
|
203
|
+
train = pd.DataFrame.sparse.from_spmatrix(
|
|
204
|
+
X[train_mask],
|
|
205
|
+
columns=var_names,
|
|
206
|
+
index=adata[train_mask].obs.index,
|
|
207
|
+
)
|
|
208
|
+
validation = pd.DataFrame.sparse.from_spmatrix(
|
|
209
|
+
X[validation_mask],
|
|
210
|
+
columns=var_names,
|
|
211
|
+
index=adata[validation_mask].obs.index,
|
|
212
|
+
)
|
|
213
|
+
full = pd.DataFrame.sparse.from_spmatrix(
|
|
214
|
+
X,
|
|
215
|
+
columns=var_names,
|
|
216
|
+
index=adata.obs.index,
|
|
217
|
+
)
|
|
218
|
+
else:
|
|
219
|
+
train = pd.DataFrame(
|
|
220
|
+
X[train_mask],
|
|
221
|
+
columns=var_names,
|
|
222
|
+
index=adata[train_mask].obs.index,
|
|
223
|
+
)
|
|
224
|
+
validation = pd.DataFrame(
|
|
225
|
+
X[validation_mask],
|
|
226
|
+
columns=var_names,
|
|
227
|
+
index=adata[validation_mask].obs.index,
|
|
228
|
+
)
|
|
229
|
+
full = pd.DataFrame(
|
|
230
|
+
X,
|
|
231
|
+
columns=var_names,
|
|
232
|
+
index=adata.obs.index,
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
for feature in features:
|
|
236
|
+
train[feature] = adata[train_mask].obs[feature]
|
|
237
|
+
validation[feature] = adata[validation_mask].obs[feature]
|
|
238
|
+
full[feature] = adata.obs[feature]
|
|
239
|
+
|
|
240
|
+
label_train = label[train_mask][:, fates_mask]
|
|
241
|
+
label_validation = label[validation_mask][:, fates_mask]
|
|
242
|
+
n_dims_response = label_train.shape[1]
|
|
243
|
+
|
|
244
|
+
if split_multi and n_dims_response > 1:
|
|
245
|
+
validation = pd.concat([validation] * n_dims_response)
|
|
246
|
+
weights_validation = label_validation.T.flatten()
|
|
247
|
+
label_validation = [i for i in range(label_validation.shape[1]) for j in range(label_validation.shape[0])]
|
|
248
|
+
|
|
249
|
+
train = pd.concat([train] * n_dims_response)
|
|
250
|
+
weights_train = label_train.T.flatten()
|
|
251
|
+
label_train = [i for i in range(label_train.shape[1]) for j in range(label_train.shape[0])]
|
|
252
|
+
else:
|
|
253
|
+
weights_train = None
|
|
254
|
+
weights_validation = None
|
|
255
|
+
|
|
256
|
+
train = Pool(data=train, label=label_train, cat_features=cat_features, weight=weights_train)
|
|
257
|
+
validation = Pool(data=validation, label=label_validation, cat_features=cat_features, weight=weights_validation)
|
|
258
|
+
full = Pool(data=full, label=label[:, fates_mask], cat_features=cat_features, weight=None)
|
|
259
|
+
|
|
260
|
+
return full, train, validation, cat_features, validation_mask, fates_mask, loss
|
|
261
|
+
|
|
262
|
+
def catboost(
|
|
263
|
+
adata: sc.AnnData,
|
|
264
|
+
obsm_key: str,
|
|
265
|
+
gs_key: str | None = None,
|
|
266
|
+
validation_key: str | None = None,
|
|
267
|
+
validation_value: str | None = None,
|
|
268
|
+
features: list[str] | None = None,
|
|
269
|
+
use_raw: bool | None = None,
|
|
270
|
+
layer: str | None = None,
|
|
271
|
+
min_size: int = 3,
|
|
272
|
+
model: Literal["regressor", "classifier"] = "regressor",
|
|
273
|
+
num_trees: int = 10000,
|
|
274
|
+
early_stopping_rounds: int = 100,
|
|
275
|
+
verbose: bool = True,
|
|
276
|
+
loss: str | None = None,
|
|
277
|
+
eval_metric: str | None = None,
|
|
278
|
+
response_transform: Literal["logit", "log1p", "sqrt", "clr"] | None = None,
|
|
279
|
+
use_gpu: bool | None = None,
|
|
280
|
+
random_state: int | None = 42,
|
|
281
|
+
prediction_key_added: str = "predicted",
|
|
282
|
+
return_model: bool = False,
|
|
283
|
+
save_model: str | os.PathLike | None = None,
|
|
284
|
+
progress_bar: bool = True,
|
|
285
|
+
catboost_dir: os.PathLike | str | None = None,
|
|
286
|
+
mask_key: str | None | Literal[False] = None,
|
|
287
|
+
clr_pseudocount: float = 1e-3,
|
|
288
|
+
**kwargs,
|
|
289
|
+
) -> sc.AnnData | tuple[sc.AnnData, CatBoostRegressor | CatBoostClassifier]:
|
|
290
|
+
"""
|
|
291
|
+
Performs CatBoost regression or classification on the data aiming to identify
|
|
292
|
+
associations between the features and the response variable.
|
|
293
|
+
|
|
294
|
+
Parameters
|
|
295
|
+
----------
|
|
296
|
+
adata : sc.AnnData
|
|
297
|
+
Annotated data matrix.
|
|
298
|
+
obsm_key : str
|
|
299
|
+
Key in `adata.obsm` to use for the response variable.
|
|
300
|
+
gs_key : str | None, optional
|
|
301
|
+
Key in `adata.obs` with `c2v.tl.gs()` parameters. Default is None.
|
|
302
|
+
validation_key : str | None, optional
|
|
303
|
+
Key in `adata.obs` to use for the train/validation split. If None, reconstructed from
|
|
304
|
+
the adata.uns[gs_key]. Default is None.
|
|
305
|
+
validation_value : str | None, optional
|
|
306
|
+
Value in `adata.obs[validation_key]` to use for the validation set. If None,
|
|
307
|
+
reconstructed from the adata.uns[gs_key]. Default is None.
|
|
308
|
+
features : list[str] | None, optional
|
|
309
|
+
List of additional adata.obs columns to use for the model (e.g. batch labels, layer names,
|
|
310
|
+
and so on). Default is None.
|
|
311
|
+
use_raw : bool | None, optional
|
|
312
|
+
Whether to use `adata.raw` for feature selection. Default is None.
|
|
313
|
+
layer : str | None, optional
|
|
314
|
+
Layer in `adata.layers` to use for feature selection. Default is None.
|
|
315
|
+
model : Literal["regressor", "classifier"], optional
|
|
316
|
+
Whether to perform regression or classification. Default is "regressor".
|
|
317
|
+
num_trees : int, optional
|
|
318
|
+
Number of trees to build. Default is 10000.
|
|
319
|
+
early_stopping_rounds : int, optional
|
|
320
|
+
Number of iterations with no improvement on validation set after which training will be stopped.
|
|
321
|
+
Default is 100.
|
|
322
|
+
verbose : bool, optional
|
|
323
|
+
Whether to print verbose output. Default is True.
|
|
324
|
+
loss : str | None, optional
|
|
325
|
+
Loss function to use. If None, set to "MultiRMSE" for multivariable regression,
|
|
326
|
+
"RMSE" for univariable regression, "MultiCrossEntropy" for multivariable classification,
|
|
327
|
+
and "CrossEntropy" for univariable classification. Default is None.
|
|
328
|
+
eval_metric : str | None, optional
|
|
329
|
+
Evaluation metric to use. Default is None.
|
|
330
|
+
response_transform : Literal["logit", "log1p", "sqrt", "clr"] | None, optional
|
|
331
|
+
Transform to apply to the response variable. Default is None.
|
|
332
|
+
use_gpu : bool | None, optional
|
|
333
|
+
Whether to use GPU for training. Default is None.
|
|
334
|
+
random_state : int | None, optional
|
|
335
|
+
Random seed for reproducibility. Default is 42.
|
|
336
|
+
prediction_key_added : str, optional
|
|
337
|
+
Key in `adata.obsm` to add the predicted values. Default is "predicted".
|
|
338
|
+
return_model : bool, optional
|
|
339
|
+
Whether to return the trained model. Default is False.
|
|
340
|
+
save_model : str | os.PathLike | None, optional
|
|
341
|
+
Path to save the trained model. Default is None.
|
|
342
|
+
progress_bar : bool, optional
|
|
343
|
+
Whether to show a progress bar during training. Default is True.
|
|
344
|
+
catboost_dir : os.PathLike | str | None, optional
|
|
345
|
+
Directory to save CatBoost training information. Default is None.
|
|
346
|
+
mask_key : str | None | Literal[False], optional
|
|
347
|
+
Key in `adata.obs` or `adata.obsm` containing a boolean mask to filter cells, by default None.
|
|
348
|
+
clr_pseudocount : float, optional
|
|
349
|
+
Pseudocount to add to expression values before CLR transformation, by default 1e-3.
|
|
350
|
+
|
|
351
|
+
**kwargs
|
|
352
|
+
Additional keyword arguments to pass to CatBoostRegressor or CatBoostClassifier.
|
|
353
|
+
|
|
354
|
+
Returns
|
|
355
|
+
-------
|
|
356
|
+
adata : sc.AnnData
|
|
357
|
+
Annotated data matrix with the predicted values added to `adata.obsm[prediction_key_added]`.
|
|
358
|
+
model : CatBoostRegressor | CatBoostClassifier
|
|
359
|
+
Trained CatBoost model.
|
|
360
|
+
"""
|
|
361
|
+
import tempfile
|
|
362
|
+
|
|
363
|
+
try:
|
|
364
|
+
from catboost.utils import get_gpu_device_count
|
|
365
|
+
from catboost import CatBoostRegressor, CatBoostClassifier
|
|
366
|
+
except ImportError:
|
|
367
|
+
raise ImportError("catboost package is not installed. Please install it using `pip install catboost`.")
|
|
368
|
+
|
|
369
|
+
# Scanpy-style logging
|
|
370
|
+
start = logg.info("training CatBoost model for associations")
|
|
371
|
+
|
|
372
|
+
if use_gpu is None:
|
|
373
|
+
use_gpu = get_gpu_device_count() > 0
|
|
374
|
+
elif use_gpu and get_gpu_device_count() == 0:
|
|
375
|
+
logg.warning("use_gpu=True, but no GPU devices are available. Setting use_gpu=False.")
|
|
376
|
+
use_gpu = False
|
|
377
|
+
|
|
378
|
+
if use_gpu:
|
|
379
|
+
task_type = "GPU"
|
|
380
|
+
else:
|
|
381
|
+
task_type = "CPU"
|
|
382
|
+
|
|
383
|
+
if adata.obsm[obsm_key].shape[1] > 1:
|
|
384
|
+
multivariable = True
|
|
385
|
+
else:
|
|
386
|
+
multivariable = False
|
|
387
|
+
|
|
388
|
+
if model == "regressor":
|
|
389
|
+
cb_model = CatBoostRegressor
|
|
390
|
+
if loss is None:
|
|
391
|
+
if multivariable:
|
|
392
|
+
loss = "MultiRMSE"
|
|
393
|
+
split_multi = False
|
|
394
|
+
else:
|
|
395
|
+
loss = "RMSE"
|
|
396
|
+
split_multi = False
|
|
397
|
+
|
|
398
|
+
elif model == "classifier":
|
|
399
|
+
cb_model = CatBoostClassifier
|
|
400
|
+
|
|
401
|
+
if loss is None:
|
|
402
|
+
loss = "MultiClass"
|
|
403
|
+
split_multi = True
|
|
404
|
+
else:
|
|
405
|
+
split_multi = False
|
|
406
|
+
|
|
407
|
+
if use_raw and layer:
|
|
408
|
+
logg.warning("use_raw and layer are mutually exclusive. Setting use_raw=False.")
|
|
409
|
+
use_raw = False
|
|
410
|
+
if use_raw is None and adata.raw:
|
|
411
|
+
logg.warning("use_raw is None and adata.raw is not None. Setting use_raw=True.")
|
|
412
|
+
use_raw = True
|
|
413
|
+
|
|
414
|
+
if layer:
|
|
415
|
+
if mask_key is None and "mask_key" in adata.uns.keys():
|
|
416
|
+
mask_key = adata.uns["mask_key"]
|
|
417
|
+
if mask_key:
|
|
418
|
+
mask = adata.obsm[mask_key][layer]
|
|
419
|
+
mask = mask.values & ~_nan_mask(adata.layers[layer])
|
|
420
|
+
else:
|
|
421
|
+
mask = ~_nan_mask(adata.layers[layer])
|
|
422
|
+
elif use_raw:
|
|
423
|
+
mask = ~_nan_mask(adata.raw.X)
|
|
424
|
+
else:
|
|
425
|
+
mask = ~_nan_mask(adata.X)
|
|
426
|
+
mask = np.array(mask)
|
|
427
|
+
if sum(~mask) > 0:
|
|
428
|
+
logg.warning(f"{int((~mask).sum())} clones won't be included in the analysis because they consist of missing values.")
|
|
429
|
+
|
|
430
|
+
if features is None:
|
|
431
|
+
features = []
|
|
432
|
+
|
|
433
|
+
full, train, validation, cat_features, validation_mask, fates_mask, loss = _prepare_pools(
|
|
434
|
+
adata[mask],
|
|
435
|
+
obsm_key=obsm_key,
|
|
436
|
+
gs_key=gs_key,
|
|
437
|
+
validation_key=validation_key,
|
|
438
|
+
validation_value=validation_value,
|
|
439
|
+
features=features,
|
|
440
|
+
use_raw=use_raw,
|
|
441
|
+
layer=layer,
|
|
442
|
+
response_transform=response_transform,
|
|
443
|
+
clr_pseudocount=clr_pseudocount,
|
|
444
|
+
split_multi=split_multi,
|
|
445
|
+
min_size=min_size,
|
|
446
|
+
loss=loss,
|
|
447
|
+
)
|
|
448
|
+
|
|
449
|
+
if eval_metric is None:
|
|
450
|
+
eval_metric = loss
|
|
451
|
+
|
|
452
|
+
with tempfile.TemporaryDirectory() as temp_dir_path:
|
|
453
|
+
if catboost_dir is None:
|
|
454
|
+
catboost_dir = temp_dir_path
|
|
455
|
+
|
|
456
|
+
cb_model = cb_model(
|
|
457
|
+
loss_function=loss,
|
|
458
|
+
eval_metric=eval_metric,
|
|
459
|
+
num_trees=num_trees,
|
|
460
|
+
early_stopping_rounds=early_stopping_rounds,
|
|
461
|
+
task_type=task_type,
|
|
462
|
+
boosting_type="Plain",
|
|
463
|
+
verbose=verbose,
|
|
464
|
+
random_seed=random_state,
|
|
465
|
+
train_dir=catboost_dir,
|
|
466
|
+
**kwargs,
|
|
467
|
+
)
|
|
468
|
+
|
|
469
|
+
if sc.settings.verbosity >= 2:
|
|
470
|
+
prefix = " "
|
|
471
|
+
progress_bar = True
|
|
472
|
+
else:
|
|
473
|
+
prefix = ""
|
|
474
|
+
|
|
475
|
+
with _catboost_tqdm(num_trees, show=progress_bar, prefix=prefix) as custom_iterator:
|
|
476
|
+
cb_model.fit(train, eval_set=validation, use_best_model=True, log_cout=custom_iterator)
|
|
477
|
+
|
|
478
|
+
logg.info(" model training finished")
|
|
479
|
+
|
|
480
|
+
if prediction_key_added in adata.obsm.keys():
|
|
481
|
+
logg.warning(f"prediction_key_added={prediction_key_added} is already in adata.obsm.keys(). Overwriting.")
|
|
482
|
+
|
|
483
|
+
if model == "classifier":
|
|
484
|
+
prediction = cb_model.predict(full, prediction_type="Probability")
|
|
485
|
+
else:
|
|
486
|
+
prediction = cb_model.predict(full)
|
|
487
|
+
|
|
488
|
+
if isinstance(adata.obsm[obsm_key], pd.DataFrame):
|
|
489
|
+
prediction_cols = adata.obsm[obsm_key].columns[fates_mask]
|
|
490
|
+
prediction = pd.DataFrame(
|
|
491
|
+
prediction,
|
|
492
|
+
columns=prediction_cols,
|
|
493
|
+
index=adata[mask].obs_names,
|
|
494
|
+
)
|
|
495
|
+
real = adata[mask].obsm[obsm_key].iloc[:, fates_mask]
|
|
496
|
+
elif sp.issparse(adata.obsm[obsm_key]):
|
|
497
|
+
prediction = sp.csr_matrix(prediction)
|
|
498
|
+
prediction_cols = np.arange(prediction.shape[1]).astype(str)[fates_mask]
|
|
499
|
+
real = adata[mask].obsm[obsm_key][:, fates_mask]
|
|
500
|
+
else:
|
|
501
|
+
prediction_cols = np.arange(prediction.shape[1]).astype(str)[fates_mask]
|
|
502
|
+
real = adata[mask].obsm[obsm_key][:, fates_mask]
|
|
503
|
+
|
|
504
|
+
logg.info(" computing SHAP")
|
|
505
|
+
raw_shap = cb_model.get_feature_importance(full, type="ShapValues")
|
|
506
|
+
non_zero_genes = np.linalg.norm(raw_shap, axis=1)[:, :-1].sum(axis=0) > 0
|
|
507
|
+
raw_shap = raw_shap[:, :, :-1][:, :, non_zero_genes]
|
|
508
|
+
|
|
509
|
+
shapdata = sc.AnnData(
|
|
510
|
+
X=sp.csr_matrix(np.linalg.norm(raw_shap, axis=1)),
|
|
511
|
+
obs=pd.DataFrame(index=adata[mask].obs_names),
|
|
512
|
+
var=pd.DataFrame(index=np.array(full.get_feature_names())[non_zero_genes]),
|
|
513
|
+
)
|
|
514
|
+
|
|
515
|
+
if response_transform == "logit":
|
|
516
|
+
shapdata.obsm[obsm_key] = np.clip(real, 0.01, 0.99).copy()
|
|
517
|
+
shapdata.obsm[obsm_key] = np.log(shapdata.obsm[obsm_key] / (1 - shapdata.obsm[obsm_key]))
|
|
518
|
+
elif response_transform == "log1p":
|
|
519
|
+
shapdata.obsm[obsm_key] = np.log1p(real).copy()
|
|
520
|
+
elif response_transform == "sqrt":
|
|
521
|
+
shapdata.obsm[obsm_key] = np.sqrt(real).copy()
|
|
522
|
+
elif response_transform == "clr":
|
|
523
|
+
shapdata.obsm[obsm_key] = _clr(real).copy()
|
|
524
|
+
else:
|
|
525
|
+
shapdata.obsm[obsm_key] = real
|
|
526
|
+
|
|
527
|
+
shapdata.obsm[f"{obsm_key}:{prediction_key_added}"] = prediction
|
|
528
|
+
|
|
529
|
+
# If we have integers, transforming predictions into estimated counts
|
|
530
|
+
if np.allclose(real, real.astype(int)):
|
|
531
|
+
shapdata.obsm[f"{obsm_key}:{prediction_key_added}"] = (
|
|
532
|
+
shapdata.obsm[f"{obsm_key}:{prediction_key_added}"].T *
|
|
533
|
+
shapdata.obsm[obsm_key].sum(axis=1)
|
|
534
|
+
).T
|
|
535
|
+
|
|
536
|
+
correlations = {}
|
|
537
|
+
mean_shap = {}
|
|
538
|
+
|
|
539
|
+
num_features = [i for i in features if i not in cat_features]
|
|
540
|
+
X_features = shapdata.var_names[shapdata.var_names.isin(adata.var_names)]
|
|
541
|
+
|
|
542
|
+
logg.info(f" calculating correlation between SHAP and expression")
|
|
543
|
+
for i, pred_col in enumerate(prediction_cols):
|
|
544
|
+
shapdata.layers[pred_col] = sp.csr_matrix(raw_shap[:, i, :])
|
|
545
|
+
mean_shap[pred_col] = np.abs(shapdata.layers[pred_col]).mean(axis=0).A[0]
|
|
546
|
+
|
|
547
|
+
if layer:
|
|
548
|
+
X = adata[mask, X_features].layers[layer]
|
|
549
|
+
elif use_raw:
|
|
550
|
+
X = adata.raw[mask, X_features].X
|
|
551
|
+
else:
|
|
552
|
+
X = adata[mask, X_features].X
|
|
553
|
+
|
|
554
|
+
X_corr = _fast_corr_cols(
|
|
555
|
+
shapdata[:, X_features].layers[pred_col],
|
|
556
|
+
X,
|
|
557
|
+
significance=False,
|
|
558
|
+
slope=False,
|
|
559
|
+
progress_bar=False,
|
|
560
|
+
)["r"]
|
|
561
|
+
|
|
562
|
+
if len(num_features) > 0:
|
|
563
|
+
obs_cor = _fast_corr_cols(
|
|
564
|
+
shapdata[:, num_features].layers[pred_col],
|
|
565
|
+
adata[mask].obs[num_features].values,
|
|
566
|
+
significance=False,
|
|
567
|
+
slope=False,
|
|
568
|
+
progress_bar=False,
|
|
569
|
+
)["r"]
|
|
570
|
+
else:
|
|
571
|
+
obs_cor = np.array([])
|
|
572
|
+
|
|
573
|
+
correlations[pred_col] = pd.Series(
|
|
574
|
+
list(X_corr) + list(obs_cor),
|
|
575
|
+
index=list(X_features) + num_features,
|
|
576
|
+
)
|
|
577
|
+
|
|
578
|
+
shapdata.varm["mean_shap"] = pd.DataFrame(mean_shap, index=shapdata.var_names)
|
|
579
|
+
shapdata.varm["gex_r"] = pd.DataFrame(correlations).reindex(shapdata.var_names)
|
|
580
|
+
shapdata.obs["validation"] = validation_mask
|
|
581
|
+
shapdata.var["mean_shap"] = shapdata.X.mean(axis=0).A[0]
|
|
582
|
+
shapdata.uns["catboost_info"] = {
|
|
583
|
+
"obsm_key": obsm_key,
|
|
584
|
+
"validation_key": validation_key,
|
|
585
|
+
"validation_value": validation_value,
|
|
586
|
+
"features": features,
|
|
587
|
+
"model": type(cb_model).__name__,
|
|
588
|
+
"num_trees": num_trees,
|
|
589
|
+
"early_stopping_rounds": early_stopping_rounds,
|
|
590
|
+
"verbose": verbose,
|
|
591
|
+
"loss": loss,
|
|
592
|
+
"eval_metric": eval_metric,
|
|
593
|
+
"response_transform": response_transform,
|
|
594
|
+
"use_gpu": use_gpu,
|
|
595
|
+
"random_state": random_state,
|
|
596
|
+
"kwargs": kwargs,
|
|
597
|
+
"layer": layer,
|
|
598
|
+
"use_raw": use_raw,
|
|
599
|
+
"fates_used": list(prediction_cols),
|
|
600
|
+
"loss_history": cb_model.get_evals_result()["learn"][loss],
|
|
601
|
+
"validation_history": cb_model.get_evals_result()["validation"][eval_metric],
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
lines = ["created AnnData with",
|
|
605
|
+
" .X matrix of mean absolute SHAP values",
|
|
606
|
+
f" .obsm['{obsm_key}'] coordinates",
|
|
607
|
+
f" .obsm['{obsm_key}:{prediction_key_added}'] predictions",
|
|
608
|
+
" .layers['<obsm_columns>'] SHAP values",
|
|
609
|
+
" .varm['mean_shap'] dataframe of absolute SHAP means per prediction column",
|
|
610
|
+
" .varm['gex_r'] dataframe of correlations between SHAP and expression",
|
|
611
|
+
" .obs['validation'] boolean vector",
|
|
612
|
+
" .uns['catboost_info'] dict of parameters"]
|
|
613
|
+
|
|
614
|
+
logg.info(" finished ({time_passed})", deep="\n".join(lines), time=start)
|
|
615
|
+
|
|
616
|
+
if save_model:
|
|
617
|
+
cb_model.save_model(save_model)
|
|
618
|
+
|
|
619
|
+
if return_model:
|
|
620
|
+
return shapdata, cb_model
|
|
621
|
+
else:
|
|
622
|
+
return shapdata
|