clone2vec 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
clone2vec/catboost.py ADDED
@@ -0,0 +1,622 @@
1
+ from __future__ import annotations
2
+
3
+ import scanpy as sc
4
+ import numpy as np
5
+ import scipy.sparse as sp
6
+ import pandas as pd
7
+
8
+ import os
9
+ import sys
10
+
11
+ from typing import Literal
12
+ from tqdm import tqdm
13
+
14
+ from .utils import _nan_mask
15
+ from .associations import _clr, _fast_corr_cols
16
+
17
+ logg = sc.logging
18
+
19
+ class _catboost_tqdm:
20
+ """
21
+ Custom progress bar for CatBoost training.
22
+
23
+ Parameters
24
+ ----------
25
+ total_iterations : int
26
+ Total number of iterations.
27
+ show : bool, optional
28
+ Whether to show the progress bar, by default True.
29
+ prefix : str, optional
30
+ Prefix for the progress bar, by default "".
31
+ """
32
+ def __init__(self, total_iterations, show: bool = True, prefix: str = ""):
33
+ if show:
34
+ self.pbar = tqdm(
35
+ total=total_iterations,
36
+ desc=f"{prefix}training",
37
+ file=sys.stdout,
38
+ )
39
+ else:
40
+ self.pbar = None
41
+ def write(self, message: str):
42
+ if self.pbar:
43
+ message_stripped = message.strip()
44
+ if message_stripped and message_stripped[0].isdigit() and ":" in message_stripped:
45
+ self.pbar.update(1)
46
+ try:
47
+ parts = message_stripped.split("\t")
48
+ self.pbar.set_postfix({
49
+ "train": float(parts[1].split(":")[-1]),
50
+ "validation": float(parts[2].split(":")[-1]),
51
+ })
52
+ except Exception:
53
+ pass
54
+ def flush(self):
55
+ if self.pbar:
56
+ sys.stdout.flush()
57
+ def close(self):
58
+ if self.pbar:
59
+ self.pbar.close()
60
+ def __enter__(self):
61
+ return self
62
+ def __exit__(self, exc_type, exc, tb):
63
+ self.close()
64
+ self.pbar = None
65
+
66
+ def _prepare_pools(
67
+ adata,
68
+ obsm_key: str,
69
+ gs_key: str | None = None,
70
+ validation_key: str | None = None,
71
+ validation_value: str | None = None,
72
+ features: list[str] | None = None,
73
+ use_raw: bool | None = None,
74
+ layer: str | None = None,
75
+ response_transform: None | Literal["logit", "log1p", "sqrt", "clr"] = None,
76
+ clr_pseudocount: float = 1e-3,
77
+ split_multi: bool = False,
78
+ min_size: int = 3,
79
+ loss: str = "MultiRMSE",
80
+ ) -> tuple["Pool", "Pool", "Pool", np.ndarray]:
81
+ """
82
+ Prepares CatBoost Pools with full, train, and validation datasets.
83
+
84
+ Parameters
85
+ ----------
86
+ adata : AnnData
87
+ AnnData object with cell metadata and embeddings.
88
+ obsm_key : str
89
+ Key in `adata.obsm` to use as the embedding.
90
+ gs_key : str | None, optional
91
+ Key in `adata.uns` to use for group splitting. Default is None.
92
+ validation_key : str | None, optional
93
+ Column in `adata.obs` to use for validation split. Default is None.
94
+ validation_value : str | None, optional
95
+ Value in `validation_key` to use for validation split. Default is None.
96
+ features : list[str] | None, optional
97
+ List of features to use as predictors. Default is None.
98
+ use_raw : bool | None, optional
99
+ Whether to use `adata.raw` for feature selection. Default is None.
100
+ layer : str | None, optional
101
+ Layer in `adata.layers` to use for feature selection. Default is None.
102
+ response_transform : None | Literal["logit", "log1p", "sqrt"], optional
103
+ Transform to apply to the response variable. Default is None.
104
+ clr_pseudocount : float, optional
105
+ Pseudocount to add to expression values before CLR transformation, by default 1e-3.
106
+ split_multi : bool, optional
107
+ Whether to split multi-dimensional response into multiple pools. Default is False.
108
+ min_size : int, optional
109
+ Minimum amount of non-zero values for each feature in the train and validation datasets. Default is 3.
110
+
111
+ Returns
112
+ -------
113
+ tuple[Pool, Pool, Pool, np.ndarray]
114
+ Full, train, validation pools and validation mask.
115
+ """
116
+ try:
117
+ from catboost import Pool
118
+ except ImportError:
119
+ raise ImportError("catboost package is not installed. Please install it using `pip install catboost`.")
120
+
121
+ if validation_key is None or validation_value is None:
122
+ if gs_key is None:
123
+ try:
124
+ validation_key = adata.uns["gs"]["obs_key"]
125
+ validation_value = adata.uns["gs"]["keys"]["validation"]
126
+ except:
127
+ raise ValueError("gs_key not found in adata.uns. Please run c2v.tl.gs() or set validation_key and validation_value.")
128
+ else:
129
+ validation_key = adata.uns[gs_key]["obs_key"]
130
+ validation_value = adata.uns[gs_key]["keys"]["validation"]
131
+ validation_mask = adata.obs[validation_key] == validation_value
132
+ train_mask = ~validation_mask
133
+
134
+ validation_mask = np.array(validation_mask)
135
+ train_mask = np.array(train_mask)
136
+
137
+ if obsm_key not in adata.obsm.keys():
138
+ raise ValueError(f"Key {obsm_key} not found in adata_train.obsm. Please provide a valid key.")
139
+
140
+ if adata.obsm[obsm_key].shape[1] > 1:
141
+ label = adata.obsm[obsm_key]
142
+ else:
143
+ label = adata.obsm[obsm_key].flatten()
144
+
145
+ if isinstance(label, pd.DataFrame):
146
+ label_names = label.columns
147
+ label = label.values
148
+ else:
149
+ label_names = np.arange(label.shape[1]).astype(str)
150
+
151
+ if split_multi:
152
+ # Checking if the label matrix is integer values
153
+ if not np.allclose(label, label.astype(int)) and ((np.round(label.sum(axis=1), 5) > 1).sum() > 0):
154
+ logg.warning("When using MultiClass loss, values must be probabilities or integers. Changing to MultiCrossEntropy.")
155
+ split_multi = False
156
+ loss = "MultiCrossEntropy"
157
+
158
+ if response_transform == "logit":
159
+ label = np.clip(label, 0.01, 0.99).copy()
160
+ label = np.log(label / (1 - label))
161
+ elif response_transform == "log1p":
162
+ label = np.log1p(label).copy()
163
+ elif response_transform == "sqrt":
164
+ label = np.sqrt(label).copy()
165
+ elif response_transform == "clr":
166
+ label = _clr(label, epsilon=clr_pseudocount).copy()
167
+ elif response_transform:
168
+ logg.warning("response_transform must be None, 'logit', 'log1p', 'sqrt', or 'clr'. Setting response_transform=None.")
169
+ response_transform = None
170
+
171
+ for feature in features:
172
+ if feature not in adata.obs.columns:
173
+ raise ValueError(f"Feature {feature} not found in adata_train.obs. Please provide a valid feature.")
174
+
175
+ cat_features = []
176
+ for feature in features:
177
+ if isinstance(adata.obs[feature], pd.CategoricalDtype) or \
178
+ isinstance(adata.obs[feature], pd.StringDtype) or \
179
+ pd.api.types.is_object_dtype(adata.obs[feature]):
180
+ cat_features.append(feature)
181
+
182
+ if layer:
183
+ X = adata.layers[layer]
184
+ var_names = adata.var_names
185
+ elif use_raw:
186
+ X = adata.raw.X
187
+ var_names = adata.raw.var_names
188
+ else:
189
+ X = adata.X
190
+ var_names = adata.var_names
191
+
192
+ fate_size_train = (label[train_mask] != 0).sum(axis=0)
193
+ fate_size_validation = (label[validation_mask] != 0).sum(axis=0)
194
+
195
+ fates_mask_train = fate_size_train >= min_size
196
+ fates_mask_validation = fate_size_validation >= min_size
197
+ fates_mask = fates_mask_train & fates_mask_validation
198
+
199
+ if sum(~fates_mask) > 0:
200
+ logg.warning(f"Removing fates with less than {min_size} clones in validation and/or train from the analysis ({', '.join(label_names[~fates_mask])}).")
201
+
202
+ if sp.issparse(adata.X):
203
+ train = pd.DataFrame.sparse.from_spmatrix(
204
+ X[train_mask],
205
+ columns=var_names,
206
+ index=adata[train_mask].obs.index,
207
+ )
208
+ validation = pd.DataFrame.sparse.from_spmatrix(
209
+ X[validation_mask],
210
+ columns=var_names,
211
+ index=adata[validation_mask].obs.index,
212
+ )
213
+ full = pd.DataFrame.sparse.from_spmatrix(
214
+ X,
215
+ columns=var_names,
216
+ index=adata.obs.index,
217
+ )
218
+ else:
219
+ train = pd.DataFrame(
220
+ X[train_mask],
221
+ columns=var_names,
222
+ index=adata[train_mask].obs.index,
223
+ )
224
+ validation = pd.DataFrame(
225
+ X[validation_mask],
226
+ columns=var_names,
227
+ index=adata[validation_mask].obs.index,
228
+ )
229
+ full = pd.DataFrame(
230
+ X,
231
+ columns=var_names,
232
+ index=adata.obs.index,
233
+ )
234
+
235
+ for feature in features:
236
+ train[feature] = adata[train_mask].obs[feature]
237
+ validation[feature] = adata[validation_mask].obs[feature]
238
+ full[feature] = adata.obs[feature]
239
+
240
+ label_train = label[train_mask][:, fates_mask]
241
+ label_validation = label[validation_mask][:, fates_mask]
242
+ n_dims_response = label_train.shape[1]
243
+
244
+ if split_multi and n_dims_response > 1:
245
+ validation = pd.concat([validation] * n_dims_response)
246
+ weights_validation = label_validation.T.flatten()
247
+ label_validation = [i for i in range(label_validation.shape[1]) for j in range(label_validation.shape[0])]
248
+
249
+ train = pd.concat([train] * n_dims_response)
250
+ weights_train = label_train.T.flatten()
251
+ label_train = [i for i in range(label_train.shape[1]) for j in range(label_train.shape[0])]
252
+ else:
253
+ weights_train = None
254
+ weights_validation = None
255
+
256
+ train = Pool(data=train, label=label_train, cat_features=cat_features, weight=weights_train)
257
+ validation = Pool(data=validation, label=label_validation, cat_features=cat_features, weight=weights_validation)
258
+ full = Pool(data=full, label=label[:, fates_mask], cat_features=cat_features, weight=None)
259
+
260
+ return full, train, validation, cat_features, validation_mask, fates_mask, loss
261
+
262
+ def catboost(
263
+ adata: sc.AnnData,
264
+ obsm_key: str,
265
+ gs_key: str | None = None,
266
+ validation_key: str | None = None,
267
+ validation_value: str | None = None,
268
+ features: list[str] | None = None,
269
+ use_raw: bool | None = None,
270
+ layer: str | None = None,
271
+ min_size: int = 3,
272
+ model: Literal["regressor", "classifier"] = "regressor",
273
+ num_trees: int = 10000,
274
+ early_stopping_rounds: int = 100,
275
+ verbose: bool = True,
276
+ loss: str | None = None,
277
+ eval_metric: str | None = None,
278
+ response_transform: Literal["logit", "log1p", "sqrt", "clr"] | None = None,
279
+ use_gpu: bool | None = None,
280
+ random_state: int | None = 42,
281
+ prediction_key_added: str = "predicted",
282
+ return_model: bool = False,
283
+ save_model: str | os.PathLike | None = None,
284
+ progress_bar: bool = True,
285
+ catboost_dir: os.PathLike | str | None = None,
286
+ mask_key: str | None | Literal[False] = None,
287
+ clr_pseudocount: float = 1e-3,
288
+ **kwargs,
289
+ ) -> sc.AnnData | tuple[sc.AnnData, CatBoostRegressor | CatBoostClassifier]:
290
+ """
291
+ Performs CatBoost regression or classification on the data aiming to identify
292
+ associations between the features and the response variable.
293
+
294
+ Parameters
295
+ ----------
296
+ adata : sc.AnnData
297
+ Annotated data matrix.
298
+ obsm_key : str
299
+ Key in `adata.obsm` to use for the response variable.
300
+ gs_key : str | None, optional
301
+ Key in `adata.obs` with `c2v.tl.gs()` parameters. Default is None.
302
+ validation_key : str | None, optional
303
+ Key in `adata.obs` to use for the train/validation split. If None, reconstructed from
304
+ the adata.uns[gs_key]. Default is None.
305
+ validation_value : str | None, optional
306
+ Value in `adata.obs[validation_key]` to use for the validation set. If None,
307
+ reconstructed from the adata.uns[gs_key]. Default is None.
308
+ features : list[str] | None, optional
309
+ List of additional adata.obs columns to use for the model (e.g. batch labels, layer names,
310
+ and so on). Default is None.
311
+ use_raw : bool | None, optional
312
+ Whether to use `adata.raw` for feature selection. Default is None.
313
+ layer : str | None, optional
314
+ Layer in `adata.layers` to use for feature selection. Default is None.
315
+ model : Literal["regressor", "classifier"], optional
316
+ Whether to perform regression or classification. Default is "regressor".
317
+ num_trees : int, optional
318
+ Number of trees to build. Default is 10000.
319
+ early_stopping_rounds : int, optional
320
+ Number of iterations with no improvement on validation set after which training will be stopped.
321
+ Default is 100.
322
+ verbose : bool, optional
323
+ Whether to print verbose output. Default is True.
324
+ loss : str | None, optional
325
+ Loss function to use. If None, set to "MultiRMSE" for multivariable regression,
326
+ "RMSE" for univariable regression, "MultiCrossEntropy" for multivariable classification,
327
+ and "CrossEntropy" for univariable classification. Default is None.
328
+ eval_metric : str | None, optional
329
+ Evaluation metric to use. Default is None.
330
+ response_transform : Literal["logit", "log1p", "sqrt", "clr"] | None, optional
331
+ Transform to apply to the response variable. Default is None.
332
+ use_gpu : bool | None, optional
333
+ Whether to use GPU for training. Default is None.
334
+ random_state : int | None, optional
335
+ Random seed for reproducibility. Default is 42.
336
+ prediction_key_added : str, optional
337
+ Key in `adata.obsm` to add the predicted values. Default is "predicted".
338
+ return_model : bool, optional
339
+ Whether to return the trained model. Default is False.
340
+ save_model : str | os.PathLike | None, optional
341
+ Path to save the trained model. Default is None.
342
+ progress_bar : bool, optional
343
+ Whether to show a progress bar during training. Default is True.
344
+ catboost_dir : os.PathLike | str | None, optional
345
+ Directory to save CatBoost training information. Default is None.
346
+ mask_key : str | None | Literal[False], optional
347
+ Key in `adata.obs` or `adata.obsm` containing a boolean mask to filter cells, by default None.
348
+ clr_pseudocount : float, optional
349
+ Pseudocount to add to expression values before CLR transformation, by default 1e-3.
350
+
351
+ **kwargs
352
+ Additional keyword arguments to pass to CatBoostRegressor or CatBoostClassifier.
353
+
354
+ Returns
355
+ -------
356
+ adata : sc.AnnData
357
+ Annotated data matrix with the predicted values added to `adata.obsm[prediction_key_added]`.
358
+ model : CatBoostRegressor | CatBoostClassifier
359
+ Trained CatBoost model.
360
+ """
361
+ import tempfile
362
+
363
+ try:
364
+ from catboost.utils import get_gpu_device_count
365
+ from catboost import CatBoostRegressor, CatBoostClassifier
366
+ except ImportError:
367
+ raise ImportError("catboost package is not installed. Please install it using `pip install catboost`.")
368
+
369
+ # Scanpy-style logging
370
+ start = logg.info("training CatBoost model for associations")
371
+
372
+ if use_gpu is None:
373
+ use_gpu = get_gpu_device_count() > 0
374
+ elif use_gpu and get_gpu_device_count() == 0:
375
+ logg.warning("use_gpu=True, but no GPU devices are available. Setting use_gpu=False.")
376
+ use_gpu = False
377
+
378
+ if use_gpu:
379
+ task_type = "GPU"
380
+ else:
381
+ task_type = "CPU"
382
+
383
+ if adata.obsm[obsm_key].shape[1] > 1:
384
+ multivariable = True
385
+ else:
386
+ multivariable = False
387
+
388
+ if model == "regressor":
389
+ cb_model = CatBoostRegressor
390
+ if loss is None:
391
+ if multivariable:
392
+ loss = "MultiRMSE"
393
+ split_multi = False
394
+ else:
395
+ loss = "RMSE"
396
+ split_multi = False
397
+
398
+ elif model == "classifier":
399
+ cb_model = CatBoostClassifier
400
+
401
+ if loss is None:
402
+ loss = "MultiClass"
403
+ split_multi = True
404
+ else:
405
+ split_multi = False
406
+
407
+ if use_raw and layer:
408
+ logg.warning("use_raw and layer are mutually exclusive. Setting use_raw=False.")
409
+ use_raw = False
410
+ if use_raw is None and adata.raw:
411
+ logg.warning("use_raw is None and adata.raw is not None. Setting use_raw=True.")
412
+ use_raw = True
413
+
414
+ if layer:
415
+ if mask_key is None and "mask_key" in adata.uns.keys():
416
+ mask_key = adata.uns["mask_key"]
417
+ if mask_key:
418
+ mask = adata.obsm[mask_key][layer]
419
+ mask = mask.values & ~_nan_mask(adata.layers[layer])
420
+ else:
421
+ mask = ~_nan_mask(adata.layers[layer])
422
+ elif use_raw:
423
+ mask = ~_nan_mask(adata.raw.X)
424
+ else:
425
+ mask = ~_nan_mask(adata.X)
426
+ mask = np.array(mask)
427
+ if sum(~mask) > 0:
428
+ logg.warning(f"{int((~mask).sum())} clones won't be included in the analysis because they consist of missing values.")
429
+
430
+ if features is None:
431
+ features = []
432
+
433
+ full, train, validation, cat_features, validation_mask, fates_mask, loss = _prepare_pools(
434
+ adata[mask],
435
+ obsm_key=obsm_key,
436
+ gs_key=gs_key,
437
+ validation_key=validation_key,
438
+ validation_value=validation_value,
439
+ features=features,
440
+ use_raw=use_raw,
441
+ layer=layer,
442
+ response_transform=response_transform,
443
+ clr_pseudocount=clr_pseudocount,
444
+ split_multi=split_multi,
445
+ min_size=min_size,
446
+ loss=loss,
447
+ )
448
+
449
+ if eval_metric is None:
450
+ eval_metric = loss
451
+
452
+ with tempfile.TemporaryDirectory() as temp_dir_path:
453
+ if catboost_dir is None:
454
+ catboost_dir = temp_dir_path
455
+
456
+ cb_model = cb_model(
457
+ loss_function=loss,
458
+ eval_metric=eval_metric,
459
+ num_trees=num_trees,
460
+ early_stopping_rounds=early_stopping_rounds,
461
+ task_type=task_type,
462
+ boosting_type="Plain",
463
+ verbose=verbose,
464
+ random_seed=random_state,
465
+ train_dir=catboost_dir,
466
+ **kwargs,
467
+ )
468
+
469
+ if sc.settings.verbosity >= 2:
470
+ prefix = " "
471
+ progress_bar = True
472
+ else:
473
+ prefix = ""
474
+
475
+ with _catboost_tqdm(num_trees, show=progress_bar, prefix=prefix) as custom_iterator:
476
+ cb_model.fit(train, eval_set=validation, use_best_model=True, log_cout=custom_iterator)
477
+
478
+ logg.info(" model training finished")
479
+
480
+ if prediction_key_added in adata.obsm.keys():
481
+ logg.warning(f"prediction_key_added={prediction_key_added} is already in adata.obsm.keys(). Overwriting.")
482
+
483
+ if model == "classifier":
484
+ prediction = cb_model.predict(full, prediction_type="Probability")
485
+ else:
486
+ prediction = cb_model.predict(full)
487
+
488
+ if isinstance(adata.obsm[obsm_key], pd.DataFrame):
489
+ prediction_cols = adata.obsm[obsm_key].columns[fates_mask]
490
+ prediction = pd.DataFrame(
491
+ prediction,
492
+ columns=prediction_cols,
493
+ index=adata[mask].obs_names,
494
+ )
495
+ real = adata[mask].obsm[obsm_key].iloc[:, fates_mask]
496
+ elif sp.issparse(adata.obsm[obsm_key]):
497
+ prediction = sp.csr_matrix(prediction)
498
+ prediction_cols = np.arange(prediction.shape[1]).astype(str)[fates_mask]
499
+ real = adata[mask].obsm[obsm_key][:, fates_mask]
500
+ else:
501
+ prediction_cols = np.arange(prediction.shape[1]).astype(str)[fates_mask]
502
+ real = adata[mask].obsm[obsm_key][:, fates_mask]
503
+
504
+ logg.info(" computing SHAP")
505
+ raw_shap = cb_model.get_feature_importance(full, type="ShapValues")
506
+ non_zero_genes = np.linalg.norm(raw_shap, axis=1)[:, :-1].sum(axis=0) > 0
507
+ raw_shap = raw_shap[:, :, :-1][:, :, non_zero_genes]
508
+
509
+ shapdata = sc.AnnData(
510
+ X=sp.csr_matrix(np.linalg.norm(raw_shap, axis=1)),
511
+ obs=pd.DataFrame(index=adata[mask].obs_names),
512
+ var=pd.DataFrame(index=np.array(full.get_feature_names())[non_zero_genes]),
513
+ )
514
+
515
+ if response_transform == "logit":
516
+ shapdata.obsm[obsm_key] = np.clip(real, 0.01, 0.99).copy()
517
+ shapdata.obsm[obsm_key] = np.log(shapdata.obsm[obsm_key] / (1 - shapdata.obsm[obsm_key]))
518
+ elif response_transform == "log1p":
519
+ shapdata.obsm[obsm_key] = np.log1p(real).copy()
520
+ elif response_transform == "sqrt":
521
+ shapdata.obsm[obsm_key] = np.sqrt(real).copy()
522
+ elif response_transform == "clr":
523
+ shapdata.obsm[obsm_key] = _clr(real).copy()
524
+ else:
525
+ shapdata.obsm[obsm_key] = real
526
+
527
+ shapdata.obsm[f"{obsm_key}:{prediction_key_added}"] = prediction
528
+
529
+ # If we have integers, transforming predictions into estimated counts
530
+ if np.allclose(real, real.astype(int)):
531
+ shapdata.obsm[f"{obsm_key}:{prediction_key_added}"] = (
532
+ shapdata.obsm[f"{obsm_key}:{prediction_key_added}"].T *
533
+ shapdata.obsm[obsm_key].sum(axis=1)
534
+ ).T
535
+
536
+ correlations = {}
537
+ mean_shap = {}
538
+
539
+ num_features = [i for i in features if i not in cat_features]
540
+ X_features = shapdata.var_names[shapdata.var_names.isin(adata.var_names)]
541
+
542
+ logg.info(f" calculating correlation between SHAP and expression")
543
+ for i, pred_col in enumerate(prediction_cols):
544
+ shapdata.layers[pred_col] = sp.csr_matrix(raw_shap[:, i, :])
545
+ mean_shap[pred_col] = np.abs(shapdata.layers[pred_col]).mean(axis=0).A[0]
546
+
547
+ if layer:
548
+ X = adata[mask, X_features].layers[layer]
549
+ elif use_raw:
550
+ X = adata.raw[mask, X_features].X
551
+ else:
552
+ X = adata[mask, X_features].X
553
+
554
+ X_corr = _fast_corr_cols(
555
+ shapdata[:, X_features].layers[pred_col],
556
+ X,
557
+ significance=False,
558
+ slope=False,
559
+ progress_bar=False,
560
+ )["r"]
561
+
562
+ if len(num_features) > 0:
563
+ obs_cor = _fast_corr_cols(
564
+ shapdata[:, num_features].layers[pred_col],
565
+ adata[mask].obs[num_features].values,
566
+ significance=False,
567
+ slope=False,
568
+ progress_bar=False,
569
+ )["r"]
570
+ else:
571
+ obs_cor = np.array([])
572
+
573
+ correlations[pred_col] = pd.Series(
574
+ list(X_corr) + list(obs_cor),
575
+ index=list(X_features) + num_features,
576
+ )
577
+
578
+ shapdata.varm["mean_shap"] = pd.DataFrame(mean_shap, index=shapdata.var_names)
579
+ shapdata.varm["gex_r"] = pd.DataFrame(correlations).reindex(shapdata.var_names)
580
+ shapdata.obs["validation"] = validation_mask
581
+ shapdata.var["mean_shap"] = shapdata.X.mean(axis=0).A[0]
582
+ shapdata.uns["catboost_info"] = {
583
+ "obsm_key": obsm_key,
584
+ "validation_key": validation_key,
585
+ "validation_value": validation_value,
586
+ "features": features,
587
+ "model": type(cb_model).__name__,
588
+ "num_trees": num_trees,
589
+ "early_stopping_rounds": early_stopping_rounds,
590
+ "verbose": verbose,
591
+ "loss": loss,
592
+ "eval_metric": eval_metric,
593
+ "response_transform": response_transform,
594
+ "use_gpu": use_gpu,
595
+ "random_state": random_state,
596
+ "kwargs": kwargs,
597
+ "layer": layer,
598
+ "use_raw": use_raw,
599
+ "fates_used": list(prediction_cols),
600
+ "loss_history": cb_model.get_evals_result()["learn"][loss],
601
+ "validation_history": cb_model.get_evals_result()["validation"][eval_metric],
602
+ }
603
+
604
+ lines = ["created AnnData with",
605
+ " .X matrix of mean absolute SHAP values",
606
+ f" .obsm['{obsm_key}'] coordinates",
607
+ f" .obsm['{obsm_key}:{prediction_key_added}'] predictions",
608
+ " .layers['<obsm_columns>'] SHAP values",
609
+ " .varm['mean_shap'] dataframe of absolute SHAP means per prediction column",
610
+ " .varm['gex_r'] dataframe of correlations between SHAP and expression",
611
+ " .obs['validation'] boolean vector",
612
+ " .uns['catboost_info'] dict of parameters"]
613
+
614
+ logg.info(" finished ({time_passed})", deep="\n".join(lines), time=start)
615
+
616
+ if save_model:
617
+ cb_model.save_model(save_model)
618
+
619
+ if return_model:
620
+ return shapdata, cb_model
621
+ else:
622
+ return shapdata