clone2vec 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- clone2vec/__init__.py +23 -0
- clone2vec/associations.py +1117 -0
- clone2vec/catboost.py +622 -0
- clone2vec/datasets.py +94 -0
- clone2vec/eigenvalues.py +187 -0
- clone2vec/embeddings.py +1204 -0
- clone2vec/integration.py +511 -0
- clone2vec/plotting.py +3540 -0
- clone2vec/preprocessing.py +660 -0
- clone2vec/seurat.py +720 -0
- clone2vec/tools.py +633 -0
- clone2vec/utils.py +618 -0
- clone2vec-0.1.0.dist-info/METADATA +86 -0
- clone2vec-0.1.0.dist-info/RECORD +16 -0
- clone2vec-0.1.0.dist-info/WHEEL +5 -0
- clone2vec-0.1.0.dist-info/top_level.txt +1 -0
clone2vec/seurat.py
ADDED
|
@@ -0,0 +1,720 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import scanpy as sc
|
|
4
|
+
import numpy as np
|
|
5
|
+
import scipy.sparse as sp
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
import sys
|
|
9
|
+
import os
|
|
10
|
+
|
|
11
|
+
from typing import Literal
|
|
12
|
+
from tqdm import tqdm
|
|
13
|
+
|
|
14
|
+
logg = sc.logging
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"integrate_data",
|
|
18
|
+
"read",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
def __dir__():
|
|
22
|
+
return sorted(__all__)
|
|
23
|
+
|
|
24
|
+
def _dgCMatrix_to_sp(
|
|
25
|
+
mat,
|
|
26
|
+
output_type: Literal["csc", "csr"] = "csc"
|
|
27
|
+
) -> sp.csr_matrix:
|
|
28
|
+
try:
|
|
29
|
+
p = np.array(mat.slots["p"])
|
|
30
|
+
i = np.array(mat.slots["i"])
|
|
31
|
+
x = np.array(mat.slots["x"])
|
|
32
|
+
dims = np.array(mat.slots["Dim"])
|
|
33
|
+
scipy_csc = sp.csc_matrix((x, i, p), shape=(dims[0], dims[1]))
|
|
34
|
+
if output_type == "csr":
|
|
35
|
+
return scipy_csc.tocsr()
|
|
36
|
+
else:
|
|
37
|
+
return scipy_csc
|
|
38
|
+
except Exception:
|
|
39
|
+
if output_type == "csr":
|
|
40
|
+
return sp.csr_matrix(np.array(mat))
|
|
41
|
+
else:
|
|
42
|
+
return sp.csc_matrix(np.array(mat))
|
|
43
|
+
|
|
44
|
+
def _dgCMatrix_to_np(Matrix):
|
|
45
|
+
data = np.array(Matrix.do_slot("x"))
|
|
46
|
+
indices = np.array(Matrix.do_slot("i"))
|
|
47
|
+
indptr = np.array(Matrix.do_slot("p"))
|
|
48
|
+
|
|
49
|
+
shape = (Matrix.do_slot("Dim")[0], Matrix.do_slot("Dim")[1])
|
|
50
|
+
csc_mat = sp.csc_matrix((data, indices, indptr), shape=shape)
|
|
51
|
+
|
|
52
|
+
return csc_mat.toarray()
|
|
53
|
+
|
|
54
|
+
def _get_RMatrix(
|
|
55
|
+
adata: sc.AnnData,
|
|
56
|
+
dummy: bool = False,
|
|
57
|
+
sparse: bool | None = None,
|
|
58
|
+
use_raw: bool = False,
|
|
59
|
+
) -> ro.rinterface.Sexp:
|
|
60
|
+
import rpy2.robjects as ro
|
|
61
|
+
from rpy2.robjects.packages import importr
|
|
62
|
+
|
|
63
|
+
Matrix = importr("Matrix")
|
|
64
|
+
|
|
65
|
+
if use_raw:
|
|
66
|
+
X = adata.raw[:, adata.var_names].X
|
|
67
|
+
else:
|
|
68
|
+
X = adata.X
|
|
69
|
+
|
|
70
|
+
X = X.T
|
|
71
|
+
n_genes, n_cells = X.shape
|
|
72
|
+
dim = ro.IntVector([n_genes, n_cells])
|
|
73
|
+
dimnames = ro.ListVector({
|
|
74
|
+
"rownames": ro.StrVector(adata.var_names),
|
|
75
|
+
"colnames": ro.StrVector(adata.obs_names),
|
|
76
|
+
})
|
|
77
|
+
|
|
78
|
+
if sparse is None:
|
|
79
|
+
sparse = sp.issparse(X)
|
|
80
|
+
|
|
81
|
+
if dummy:
|
|
82
|
+
if sparse:
|
|
83
|
+
return Matrix.sparseMatrix(
|
|
84
|
+
i = ro.IntVector([]),
|
|
85
|
+
p = ro.IntVector([0] * (n_cells + 1)),
|
|
86
|
+
x = ro.FloatVector([]),
|
|
87
|
+
dim = dim,
|
|
88
|
+
dimnames = dimnames,
|
|
89
|
+
)
|
|
90
|
+
else:
|
|
91
|
+
return ro.r.matrix(
|
|
92
|
+
ro.FloatVector(np.zeros(n_genes * n_cells)),
|
|
93
|
+
nrow=n_genes, ncol=n_cells,
|
|
94
|
+
byrow=False, dimnames=dimnames,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
if sparse:
|
|
98
|
+
X = X.tocsc() if sp.issparse(X) else sp.csc_matrix(X)
|
|
99
|
+
return Matrix.sparseMatrix(
|
|
100
|
+
i = ro.IntVector((X.indices + 1).tolist()),
|
|
101
|
+
p = ro.IntVector(X.indptr),
|
|
102
|
+
x = ro.FloatVector(X.data),
|
|
103
|
+
dim = dim,
|
|
104
|
+
dimnames = dimnames,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
data_vec = ro.FloatVector(np.asarray(X, dtype=float).flatten(order="F"))
|
|
108
|
+
return ro.r.matrix(
|
|
109
|
+
data_vec,
|
|
110
|
+
nrow=n_genes,
|
|
111
|
+
ncol=n_cells,
|
|
112
|
+
byrow=False,
|
|
113
|
+
dimnames=dimnames,
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
def read(
|
|
117
|
+
path: str | os.PathLike,
|
|
118
|
+
qs: bool = False,
|
|
119
|
+
assays: str | list[str] | None = None,
|
|
120
|
+
use_slots: Literal["counts", "data"] | list[Literal["counts", "data"]] = ["counts", "data"],
|
|
121
|
+
graphs: bool = True,
|
|
122
|
+
dimreds: bool = True,
|
|
123
|
+
) -> "mudata.MuData":
|
|
124
|
+
"""
|
|
125
|
+
Reads .rds-file or .qsave-file with Seurat object and returns AnnData or MuData object.
|
|
126
|
+
|
|
127
|
+
Parameters
|
|
128
|
+
----------
|
|
129
|
+
path: str | os.PathLike
|
|
130
|
+
Path to the file.
|
|
131
|
+
qs: bool, optional
|
|
132
|
+
Whether to use qs package to read the file. Default is False.
|
|
133
|
+
assays: str | list[str] | None, optional
|
|
134
|
+
Assays to process. If None, all assays are processed.
|
|
135
|
+
use_slots: Literal["counts", "data"] | list[Literal["counts", "data"]], optional
|
|
136
|
+
Slots to use for counts. Can be "counts" or "data". Default is ["counts", "data"].
|
|
137
|
+
graphs: bool, optional
|
|
138
|
+
Whether to process graphs. Default is True.
|
|
139
|
+
dimreds: bool, optional
|
|
140
|
+
Whether to process dimensionality reductions. Default is True.
|
|
141
|
+
|
|
142
|
+
Returns
|
|
143
|
+
-------
|
|
144
|
+
mudata.MuData
|
|
145
|
+
MuData object.
|
|
146
|
+
"""
|
|
147
|
+
logg.warning("This is an rpy2-wrapper around Seurat. This function is very RAM intensive and has a poor garbage collection.")
|
|
148
|
+
|
|
149
|
+
import gc
|
|
150
|
+
import rpy2
|
|
151
|
+
import rpy2.robjects as ro
|
|
152
|
+
import mudata
|
|
153
|
+
|
|
154
|
+
mudata.set_options(pull_on_update=False)
|
|
155
|
+
|
|
156
|
+
from rpy2.robjects.packages import importr
|
|
157
|
+
from rpy2.robjects.conversion import localconverter
|
|
158
|
+
import rpy2.rinterface_lib.callbacks as callbacks
|
|
159
|
+
from rpy2.robjects import pandas2ri
|
|
160
|
+
|
|
161
|
+
class _RCallbacks():
|
|
162
|
+
def __init__(self, callback):
|
|
163
|
+
self.callback = callback
|
|
164
|
+
def __enter__(self):
|
|
165
|
+
self._print = callbacks.consolewrite_print
|
|
166
|
+
self._warnerror = callbacks.consolewrite_warnerror
|
|
167
|
+
callbacks.consolewrite_print = (lambda x: None)
|
|
168
|
+
callbacks.consolewrite_warnerror = self.callback
|
|
169
|
+
return self
|
|
170
|
+
def __exit__(self, exc_type, exc, tb):
|
|
171
|
+
callbacks.consolewrite_print = self._print
|
|
172
|
+
callbacks.consolewrite_warnerror = self._warnerror
|
|
173
|
+
|
|
174
|
+
start = logg.info(f"creating MuData object from Seurat object")
|
|
175
|
+
if isinstance(use_slots, str):
|
|
176
|
+
use_slots = [use_slots]
|
|
177
|
+
|
|
178
|
+
try:
|
|
179
|
+
with ro.local_context():
|
|
180
|
+
with _RCallbacks(lambda x: None):
|
|
181
|
+
Seurat = importr("Seurat")
|
|
182
|
+
base = importr("base")
|
|
183
|
+
|
|
184
|
+
logg.info(" reading Seurat object")
|
|
185
|
+
if qs:
|
|
186
|
+
qs = importr("qs")
|
|
187
|
+
seurat_object = qs.qread(path)
|
|
188
|
+
else:
|
|
189
|
+
seurat_object = base.readRDS(path)
|
|
190
|
+
|
|
191
|
+
logg.info(" getting .obs")
|
|
192
|
+
with localconverter(ro.default_converter + pandas2ri.converter):
|
|
193
|
+
obs = seurat_object.slots["meta.data"]
|
|
194
|
+
obs = pd.DataFrame(obs)
|
|
195
|
+
obs_names = obs.index
|
|
196
|
+
|
|
197
|
+
available_assays = list(base.names(seurat_object.slots["assays"]))
|
|
198
|
+
if assays is None:
|
|
199
|
+
assays_to_process = available_assays
|
|
200
|
+
elif isinstance(assays, str):
|
|
201
|
+
assays_to_process = [assays]
|
|
202
|
+
else:
|
|
203
|
+
assays_to_process = assays
|
|
204
|
+
valid_assays = [a for a in assays_to_process if a in available_assays]
|
|
205
|
+
if len(valid_assays) == 0:
|
|
206
|
+
raise ValueError(f"no valid assays found, available: {available_assays}")
|
|
207
|
+
if len(valid_assays) < len(assays_to_process):
|
|
208
|
+
absent_assays = [a for a in assays_to_process if not (a in available_assays)]
|
|
209
|
+
logg.warning(f"following assays are not available: {absent_assays}")
|
|
210
|
+
|
|
211
|
+
adatas = {}
|
|
212
|
+
for assay_name in valid_assays:
|
|
213
|
+
logg.info(f" processing assay {assay_name}")
|
|
214
|
+
|
|
215
|
+
assay_obj = seurat_object.slots["assays"].rx2(assay_name)
|
|
216
|
+
assay_class = tuple(assay_obj.rclass)[0]
|
|
217
|
+
|
|
218
|
+
collected_matrices = {}
|
|
219
|
+
for slot_type in use_slots:
|
|
220
|
+
mat = None
|
|
221
|
+
try:
|
|
222
|
+
if assay_class == "Assay5":
|
|
223
|
+
layers = assay_obj.slots["layers"]
|
|
224
|
+
layer_names = list(base.names(layers))
|
|
225
|
+
if slot_type in layer_names:
|
|
226
|
+
mat = layers.rx2(slot_type)
|
|
227
|
+
else:
|
|
228
|
+
if slot_type in assay_obj.slots.keys():
|
|
229
|
+
mat = assay_obj.slots[slot_type]
|
|
230
|
+
except Exception:
|
|
231
|
+
pass
|
|
232
|
+
|
|
233
|
+
if mat is not None and not isinstance(mat, rpy2.rinterface_lib.sexp.NULLType):
|
|
234
|
+
collected_matrices[slot_type] = mat
|
|
235
|
+
|
|
236
|
+
if len(collected_matrices) == 0:
|
|
237
|
+
logg.warning(f"no matrices found for assay {assay_name}")
|
|
238
|
+
continue
|
|
239
|
+
|
|
240
|
+
adata_structure = {
|
|
241
|
+
"X": None,
|
|
242
|
+
"raw.X": None,
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
if len(collected_matrices) == 1:
|
|
246
|
+
if len(use_slots) == 2:
|
|
247
|
+
logg.warning(f"only one slot found for assay {assay_name}, using {slot_type} as .X")
|
|
248
|
+
|
|
249
|
+
slot_type = list(collected_matrices.keys())[0]
|
|
250
|
+
r_X = collected_matrices[slot_type]
|
|
251
|
+
X = _dgCMatrix_to_sp(r_X).T
|
|
252
|
+
adata_structure["X"] = slot_type
|
|
253
|
+
|
|
254
|
+
logg.hint(f"using slot '{slot_type}' as .X")
|
|
255
|
+
else:
|
|
256
|
+
r_X = collected_matrices["data"]
|
|
257
|
+
X = _dgCMatrix_to_sp(r_X).T
|
|
258
|
+
adata_structure["X"] = "data"
|
|
259
|
+
|
|
260
|
+
r_Xraw = collected_matrices["counts"]
|
|
261
|
+
Xraw = _dgCMatrix_to_sp(r_Xraw).T
|
|
262
|
+
adata_structure["raw.X"] = "counts"
|
|
263
|
+
|
|
264
|
+
logg.hint("using slot 'data' as .X and slot 'counts' as .raw.X")
|
|
265
|
+
|
|
266
|
+
gene_names = {}
|
|
267
|
+
if assay_class == "Assay5":
|
|
268
|
+
features = pd.DataFrame(
|
|
269
|
+
np.array(assay_obj.slots["features"]),
|
|
270
|
+
index=np.array(base.rownames(assay_obj.slots["features"])),
|
|
271
|
+
columns=np.array(base.colnames(assay_obj.slots["features"])),
|
|
272
|
+
).astype(bool)
|
|
273
|
+
|
|
274
|
+
gene_names["X"] = features.index[features[adata_structure["X"]]]
|
|
275
|
+
if adata_structure["raw.X"] is not None:
|
|
276
|
+
gene_names["raw.X"] = features.index[features[adata_structure["raw.X"]]]
|
|
277
|
+
else:
|
|
278
|
+
gene_names["X"] = base.rownames(assay_obj.slots[adata_structure["X"]])
|
|
279
|
+
if adata_structure["raw.X"] is not None:
|
|
280
|
+
gene_names["raw.X"] = base.rownames(assay_obj.slots[adata_structure["raw.X"]])
|
|
281
|
+
|
|
282
|
+
adatas[assay_name] = sc.AnnData(
|
|
283
|
+
X=X,
|
|
284
|
+
obs=pd.DataFrame(index=obs_names),
|
|
285
|
+
var=pd.DataFrame(index=gene_names["X"]),
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
if adata_structure["raw.X"] is not None:
|
|
289
|
+
adatas[assay_name].raw = sc.AnnData(
|
|
290
|
+
X=Xraw,
|
|
291
|
+
obs=pd.DataFrame(index=obs_names),
|
|
292
|
+
var=pd.DataFrame(index=gene_names["raw.X"]),
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
mdata = mudata.MuData(adatas)
|
|
296
|
+
mdata.obs = obs
|
|
297
|
+
|
|
298
|
+
if dimreds:
|
|
299
|
+
logg.info(" collecting dimensionality reductions")
|
|
300
|
+
if "reductions" in seurat_object.slots.keys():
|
|
301
|
+
reductions = seurat_object.slots["reductions"]
|
|
302
|
+
if len(reductions) > 0:
|
|
303
|
+
for red_name in base.names(reductions):
|
|
304
|
+
red_obj = reductions.rx2(red_name)
|
|
305
|
+
emb = np.array(red_obj.slots["cell.embeddings"])
|
|
306
|
+
mdata.obsm[red_name] = emb
|
|
307
|
+
mdata.uns[red_name] = {
|
|
308
|
+
"assay.used": np.array(red_obj.slots["assay.used"]),
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
if graphs:
|
|
312
|
+
logg.info(" collecting graphs")
|
|
313
|
+
if "graphs" in seurat_object.slots.keys():
|
|
314
|
+
graphs = seurat_object.slots["graphs"]
|
|
315
|
+
if len(graphs) > 0:
|
|
316
|
+
for graph_name in base.names(graphs):
|
|
317
|
+
graph_obj = graphs.rx2(graph_name)
|
|
318
|
+
mdata.obsp[graph_name] = _dgCMatrix_to_sp(graph_obj, output_type="csr")
|
|
319
|
+
|
|
320
|
+
if "commands" in seurat_object.slots.keys():
|
|
321
|
+
mdata.uns["seurat_commands"] = []
|
|
322
|
+
for command in seurat_object.slots["commands"]:
|
|
323
|
+
mdata.uns["seurat_commands"].append("\n".join(np.array(command.slots["call.string"])))
|
|
324
|
+
|
|
325
|
+
# Garbage collection, maybe it will help
|
|
326
|
+
gc.collect()
|
|
327
|
+
del seurat_object
|
|
328
|
+
ro.r("gc(verbose = FALSE)")
|
|
329
|
+
gc.collect()
|
|
330
|
+
|
|
331
|
+
lines = [
|
|
332
|
+
"created MuData with",
|
|
333
|
+
f" .mod with different assays",
|
|
334
|
+
]
|
|
335
|
+
if len(mdata.obsp.keys()) > 0:
|
|
336
|
+
lines.append(f" .obsp with different graphs")
|
|
337
|
+
if len(mdata.obsm.keys()) > 0:
|
|
338
|
+
lines.append(f" .obsm with different dimensionality reductions")
|
|
339
|
+
|
|
340
|
+
logg.info(" finished ({time_passed})", deep="\n".join(lines), time=start)
|
|
341
|
+
return mdata
|
|
342
|
+
finally:
|
|
343
|
+
gc.collect()
|
|
344
|
+
with _RCallbacks(lambda x: None):
|
|
345
|
+
ro.r("gc(verbose = FALSE)")
|
|
346
|
+
gc.collect()
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def integrate_data(
|
|
350
|
+
adata: sc.AnnData,
|
|
351
|
+
batch_key: str,
|
|
352
|
+
method: Literal["cca", "rpca"] = "cca",
|
|
353
|
+
layer_name: str = "integrated",
|
|
354
|
+
scale_integrated: bool = True,
|
|
355
|
+
scale_max_value: float = 10.,
|
|
356
|
+
l2_norm: bool = True,
|
|
357
|
+
n_comps: int = 30,
|
|
358
|
+
k_anchor: int = 5,
|
|
359
|
+
k_filter: int = 200,
|
|
360
|
+
k_score: int = 30,
|
|
361
|
+
max_features: int = 200,
|
|
362
|
+
n_trees: int = 50,
|
|
363
|
+
eps: float = 0,
|
|
364
|
+
k_weight: int = 100,
|
|
365
|
+
progress_bar: bool = True,
|
|
366
|
+
only_anchors: bool = False,
|
|
367
|
+
mnn_graph_key: str = "seurat_mnn",
|
|
368
|
+
uns_key: str = "seurat_integration",
|
|
369
|
+
) -> ro.rinterface.Sexp:
|
|
370
|
+
"""
|
|
371
|
+
Performs single-cell data integration via Seurat's CCA or RPCA approach.
|
|
372
|
+
|
|
373
|
+
Parameters
|
|
374
|
+
----------
|
|
375
|
+
adata
|
|
376
|
+
Annotated data matrix.
|
|
377
|
+
batch_key
|
|
378
|
+
Key in `adata.obs` that contains batch information.
|
|
379
|
+
method
|
|
380
|
+
Integration method to use. Either "cca" or "rpca".
|
|
381
|
+
layer_name
|
|
382
|
+
Name of the layer to store the integrated expression matrix.
|
|
383
|
+
scale_integrated
|
|
384
|
+
Whether to scale the integrated expression matrix.
|
|
385
|
+
scale_max_value
|
|
386
|
+
Maximum value to scale the integrated expression matrix.
|
|
387
|
+
l2_norm
|
|
388
|
+
Perform L2 normalization on the CCA cell embeddings after dimensional reduction.
|
|
389
|
+
n_comps
|
|
390
|
+
Which dimensions to use from the CCA to specify the neighbor search space.
|
|
391
|
+
k_anchor
|
|
392
|
+
How many neighbors (k) to use when picking anchors.
|
|
393
|
+
k_filter
|
|
394
|
+
How many neighbors (k) to use when filtering anchors.
|
|
395
|
+
k_score
|
|
396
|
+
How many neighbors (k) to use when scoring anchors.
|
|
397
|
+
max_features
|
|
398
|
+
The maximum number of features to use when specifying the neighborhood search space in the anchor filtering.
|
|
399
|
+
n_trees
|
|
400
|
+
More trees gives higher precision when using annoy approximate nearest neighbor search.
|
|
401
|
+
eps
|
|
402
|
+
Error bound on the neighbor finding algorithm.
|
|
403
|
+
k_weight
|
|
404
|
+
Number of neighbors to consider when weighting anchors.
|
|
405
|
+
progress_bar
|
|
406
|
+
Whether to show a progress bar.
|
|
407
|
+
return_anchors
|
|
408
|
+
Whether to return the anchors.
|
|
409
|
+
|
|
410
|
+
Returns
|
|
411
|
+
-------
|
|
412
|
+
anchors
|
|
413
|
+
Seurat anchors object.
|
|
414
|
+
"""
|
|
415
|
+
logg.warning("This is an rpy2-wrapper around Seurat. This function is very RAM intensive and has a poor garbage collection.")
|
|
416
|
+
|
|
417
|
+
import gc
|
|
418
|
+
import rpy2.robjects as ro
|
|
419
|
+
import rpy2.rinterface_lib.callbacks as callbacks
|
|
420
|
+
from rpy2.robjects.packages import importr
|
|
421
|
+
from contextlib import nullcontext
|
|
422
|
+
|
|
423
|
+
class _RCallbacks():
|
|
424
|
+
def __init__(self, callback):
|
|
425
|
+
self.callback = callback
|
|
426
|
+
def __enter__(self):
|
|
427
|
+
self._print = callbacks.consolewrite_print
|
|
428
|
+
self._warnerror = callbacks.consolewrite_warnerror
|
|
429
|
+
callbacks.consolewrite_print = (lambda x: None)
|
|
430
|
+
callbacks.consolewrite_warnerror = self.callback
|
|
431
|
+
return self
|
|
432
|
+
def __exit__(self, exc_type, exc, tb):
|
|
433
|
+
callbacks.consolewrite_print = self._print
|
|
434
|
+
callbacks.consolewrite_warnerror = self._warnerror
|
|
435
|
+
|
|
436
|
+
try:
|
|
437
|
+
with ro.local_context():
|
|
438
|
+
with _RCallbacks(lambda x: None):
|
|
439
|
+
Seurat = importr("Seurat")
|
|
440
|
+
SeuratObject = importr("SeuratObject")
|
|
441
|
+
|
|
442
|
+
try:
|
|
443
|
+
batch = ro.StrVector(adata.obs[batch_key].astype(str).values)
|
|
444
|
+
batch.names = ro.StrVector(adata.obs_names)
|
|
445
|
+
except KeyError:
|
|
446
|
+
raise ValueError(f"Batch key {batch_key} not found in adata.obs.")
|
|
447
|
+
|
|
448
|
+
if sp.issparse(adata.X):
|
|
449
|
+
raise ValueError("adata.X must be dense scaled matrix.")
|
|
450
|
+
|
|
451
|
+
if adata.raw is None:
|
|
452
|
+
raise ValueError("adata.raw must contain log1p-transformed counts.")
|
|
453
|
+
|
|
454
|
+
start = logg.info(f"integrating data with Seurat {method.upper()} approach")
|
|
455
|
+
|
|
456
|
+
# Avoiding conflicts with Seurat's convention of using "-" instead of "_"
|
|
457
|
+
var_names = adata.var_names
|
|
458
|
+
underscores = sum(["_" in i for i in var_names]) > 0
|
|
459
|
+
if underscores:
|
|
460
|
+
raw_var_names = adata.raw.var_names
|
|
461
|
+
var_names_mod = [i.replace("_", "-") for i in var_names]
|
|
462
|
+
raw_var_names_mod = [i.replace("_", "-") for i in adata.raw.var_names]
|
|
463
|
+
adata.var_names = var_names_mod
|
|
464
|
+
adata.raw.var.index = raw_var_names_mod
|
|
465
|
+
|
|
466
|
+
seurat_obj = SeuratObject.CreateSeuratObject(
|
|
467
|
+
counts=_get_RMatrix(adata, sparse=True, dummy=True),
|
|
468
|
+
project="AnnData",
|
|
469
|
+
)
|
|
470
|
+
|
|
471
|
+
seurat_obj = SeuratObject.SetAssayData(
|
|
472
|
+
object = seurat_obj,
|
|
473
|
+
slot = "data",
|
|
474
|
+
new_data = _get_RMatrix(adata, sparse=True, use_raw=True, dummy=False),
|
|
475
|
+
)
|
|
476
|
+
|
|
477
|
+
seurat_obj = SeuratObject.SetAssayData(
|
|
478
|
+
object = seurat_obj,
|
|
479
|
+
slot = "scale.data",
|
|
480
|
+
new_data = _get_RMatrix(adata, sparse=False, use_raw=False, dummy=False),
|
|
481
|
+
)
|
|
482
|
+
|
|
483
|
+
seurat_obj = SeuratObject.AddMetaData(
|
|
484
|
+
seurat_obj,
|
|
485
|
+
metadata=batch,
|
|
486
|
+
col_name="batch",
|
|
487
|
+
)
|
|
488
|
+
|
|
489
|
+
seurat_obj_list = Seurat.SplitObject(seurat_obj, split_by="batch")
|
|
490
|
+
features = ro.StrVector(adata.var_names.values)
|
|
491
|
+
|
|
492
|
+
ro.globalenv["seurat.obj.list"] = seurat_obj_list
|
|
493
|
+
ro.globalenv["features"] = features
|
|
494
|
+
|
|
495
|
+
if method == "rpca":
|
|
496
|
+
with _RCallbacks(lambda x: None):
|
|
497
|
+
ro.r("""
|
|
498
|
+
seurat.obj.list <- lapply(X = seurat.obj.list, FUN = function(x) {
|
|
499
|
+
x <- RunPCA(x, features = features, verbose = FALSE, npcs = n_comps)
|
|
500
|
+
})
|
|
501
|
+
""".replace("n_comps", str(n_comps)))
|
|
502
|
+
|
|
503
|
+
if sc.settings.verbosity >= 2:
|
|
504
|
+
prefix = " "
|
|
505
|
+
progress_bar = True
|
|
506
|
+
else:
|
|
507
|
+
prefix = ""
|
|
508
|
+
|
|
509
|
+
n_batches = len(set(adata.obs[batch_key].values))
|
|
510
|
+
total_pairs = int(n_batches * (n_batches - 1) / 2)
|
|
511
|
+
|
|
512
|
+
if progress_bar:
|
|
513
|
+
cm = tqdm(
|
|
514
|
+
total=total_pairs,
|
|
515
|
+
desc=f"{prefix}finding anchors",
|
|
516
|
+
unit="pair",
|
|
517
|
+
file=sys.stdout,
|
|
518
|
+
)
|
|
519
|
+
else:
|
|
520
|
+
cm = nullcontext()
|
|
521
|
+
|
|
522
|
+
with cm as pbar:
|
|
523
|
+
if progress_bar:
|
|
524
|
+
pbar.first_iteration = True
|
|
525
|
+
pbar.prev_message = ""
|
|
526
|
+
|
|
527
|
+
def FindAnchorsCallback(s):
|
|
528
|
+
if progress_bar:
|
|
529
|
+
s_clean = s.strip().replace("\n", "")
|
|
530
|
+
if ("Running CCA" in s_clean) or ("Projecting new data onto SVD" in s_clean):
|
|
531
|
+
if not ("Projecting new data onto SVD" in pbar.prev_message):
|
|
532
|
+
if pbar.first_iteration:
|
|
533
|
+
pbar.first_iteration = False
|
|
534
|
+
else:
|
|
535
|
+
pbar.update(1)
|
|
536
|
+
if "is deprecated" not in s_clean:
|
|
537
|
+
pbar.set_postfix_str(s_clean)
|
|
538
|
+
pbar.prev_message = s_clean
|
|
539
|
+
|
|
540
|
+
with _RCallbacks(FindAnchorsCallback):
|
|
541
|
+
ro.r(f"""
|
|
542
|
+
integration.anchors <- FindIntegrationAnchors(
|
|
543
|
+
object.list = seurat.obj.list,
|
|
544
|
+
normalization.method = 'LogNormalize',
|
|
545
|
+
anchor.features = features,
|
|
546
|
+
scale = FALSE,
|
|
547
|
+
reduction = '{method}',
|
|
548
|
+
l2.norm = {str(l2_norm).upper()},
|
|
549
|
+
dims = 1:{n_comps},
|
|
550
|
+
k.anchor = {k_anchor},
|
|
551
|
+
k.filter = {k_filter},
|
|
552
|
+
k.score = {k_score},
|
|
553
|
+
max.features = {max_features},
|
|
554
|
+
n.trees = {n_trees},
|
|
555
|
+
eps = {eps}
|
|
556
|
+
)
|
|
557
|
+
""")
|
|
558
|
+
|
|
559
|
+
if progress_bar:
|
|
560
|
+
pbar.update(1)
|
|
561
|
+
pbar.set_postfix_str("")
|
|
562
|
+
|
|
563
|
+
# Reconstructing MNN matrix
|
|
564
|
+
with _RCallbacks(lambda x: None):
|
|
565
|
+
cell_names = {}
|
|
566
|
+
for i, seurat_obj_i in enumerate(seurat_obj_list):
|
|
567
|
+
cell_names[i + 1] = np.array(ro.r["colnames"](seurat_obj_i))
|
|
568
|
+
anchors = pd.DataFrame(
|
|
569
|
+
ro.r("integration.anchors@anchors"),
|
|
570
|
+
index=["cell1", "cell2", "weight", "dataset1", "dataset2"],
|
|
571
|
+
).T
|
|
572
|
+
anchors["cell1"] = anchors.apply(lambda x: cell_names[int(x.dataset1)][int(x.cell1) - 1], axis=1)
|
|
573
|
+
anchors["cell2"] = anchors.apply(lambda x: cell_names[int(x.dataset2)][int(x.cell2) - 1], axis=1)
|
|
574
|
+
del anchors["dataset1"], anchors["dataset2"]
|
|
575
|
+
|
|
576
|
+
obs_index = pd.Index(adata.obs_names)
|
|
577
|
+
idx1 = obs_index.get_indexer(anchors["cell1"].values).astype(np.int64)
|
|
578
|
+
idx2 = obs_index.get_indexer(anchors["cell2"].values).astype(np.int64)
|
|
579
|
+
|
|
580
|
+
rows = np.concatenate([idx1, idx2])
|
|
581
|
+
cols = np.concatenate([idx2, idx1])
|
|
582
|
+
values = np.concatenate([anchors["weight"].values, anchors["weight"].values])
|
|
583
|
+
adata.obsp[mnn_graph_key] = sp.csr_matrix((values, (rows, cols)), shape=(adata.n_obs, adata.n_obs))
|
|
584
|
+
|
|
585
|
+
adata.uns[uns_key] = {
|
|
586
|
+
"method": method,
|
|
587
|
+
"l2_norm": l2_norm,
|
|
588
|
+
"n_comps": n_comps,
|
|
589
|
+
"k.anchor": k_anchor,
|
|
590
|
+
"k.filter": k_filter,
|
|
591
|
+
"k.score": k_score,
|
|
592
|
+
"max.features": max_features,
|
|
593
|
+
"n.trees": n_trees,
|
|
594
|
+
"eps": eps,
|
|
595
|
+
"batch_key": batch_key,
|
|
596
|
+
"graph_key": mnn_graph_key,
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
if only_anchors:
|
|
600
|
+
adata.uns[uns_key]["mode"] = "anchors"
|
|
601
|
+
|
|
602
|
+
# Additional garbage collection
|
|
603
|
+
try:
|
|
604
|
+
del seurat_obj, seurat_obj_list, features
|
|
605
|
+
except NameError:
|
|
606
|
+
pass
|
|
607
|
+
|
|
608
|
+
gc.collect()
|
|
609
|
+
for i in ["seurat.obj.list", "features", "integration.anchors"]:
|
|
610
|
+
if i in ro.globalenv:
|
|
611
|
+
del ro.globalenv[i]
|
|
612
|
+
|
|
613
|
+
ro.r("gc(verbose = FALSE)")
|
|
614
|
+
gc.collect()
|
|
615
|
+
|
|
616
|
+
lines = [
|
|
617
|
+
"added",
|
|
618
|
+
f" .obsp['{mnn_graph_key}'] MNN graph",
|
|
619
|
+
f" .uns['{uns_key}'] MNN search parameters"
|
|
620
|
+
]
|
|
621
|
+
logg.info(" finished ({time_passed})", deep="\n".join(lines), time=start)
|
|
622
|
+
return None
|
|
623
|
+
|
|
624
|
+
total_integrations = n_batches - 1
|
|
625
|
+
if progress_bar:
|
|
626
|
+
cm = tqdm(
|
|
627
|
+
total=total_integrations,
|
|
628
|
+
desc=f"{prefix}integrating data",
|
|
629
|
+
unit="int",
|
|
630
|
+
file=sys.stdout,
|
|
631
|
+
)
|
|
632
|
+
else:
|
|
633
|
+
cm = nullcontext()
|
|
634
|
+
|
|
635
|
+
with cm as pbar:
|
|
636
|
+
if progress_bar:
|
|
637
|
+
pbar.first_iteration = True
|
|
638
|
+
|
|
639
|
+
def IntegrateDataCallback(s):
|
|
640
|
+
if progress_bar:
|
|
641
|
+
s_clean = s.strip()
|
|
642
|
+
if "Merging" in s_clean:
|
|
643
|
+
if pbar.first_iteration:
|
|
644
|
+
pbar.first_iteration = False
|
|
645
|
+
else:
|
|
646
|
+
pbar.update(1)
|
|
647
|
+
pbar.set_postfix_str(s_clean)
|
|
648
|
+
|
|
649
|
+
with _RCallbacks(IntegrateDataCallback):
|
|
650
|
+
ro.r(f"""
|
|
651
|
+
seurat.integrated <- IntegrateData(
|
|
652
|
+
anchorset = integration.anchors,
|
|
653
|
+
normalization.method = "LogNormalize",
|
|
654
|
+
features.to.integrate = features,
|
|
655
|
+
dims = 1:{n_comps},
|
|
656
|
+
k.weight = {k_weight}
|
|
657
|
+
)
|
|
658
|
+
""")
|
|
659
|
+
|
|
660
|
+
if progress_bar:
|
|
661
|
+
pbar.update(1)
|
|
662
|
+
pbar.set_postfix_str("")
|
|
663
|
+
|
|
664
|
+
seurat_integrated = ro.r("seurat.integrated")
|
|
665
|
+
integrated_matrix = SeuratObject.GetAssayData(
|
|
666
|
+
seurat_integrated,
|
|
667
|
+
assay="integrated",
|
|
668
|
+
slot="data",
|
|
669
|
+
)
|
|
670
|
+
var_names_integrated = list(ro.r["rownames"](integrated_matrix))
|
|
671
|
+
obs_names_integrated = list(ro.r["colnames"](integrated_matrix))
|
|
672
|
+
integrated_matrix = _dgCMatrix_to_np(integrated_matrix).T
|
|
673
|
+
integrated_matrix = pd.DataFrame(
|
|
674
|
+
integrated_matrix,
|
|
675
|
+
columns=var_names_integrated,
|
|
676
|
+
index=obs_names_integrated,
|
|
677
|
+
)
|
|
678
|
+
adata.layers[layer_name] = integrated_matrix.loc[
|
|
679
|
+
adata.obs_names, adata.var_names
|
|
680
|
+
].values.astype("float32")
|
|
681
|
+
if scale_integrated:
|
|
682
|
+
sc.pp.scale(adata, layer=layer_name, max_value=scale_max_value)
|
|
683
|
+
|
|
684
|
+
adata.uns[uns_key]["mode"] = "anchors + integration"
|
|
685
|
+
adata.uns[uns_key]["k.weight"] = k_weight
|
|
686
|
+
|
|
687
|
+
if underscores:
|
|
688
|
+
adata.var_names = var_names
|
|
689
|
+
adata.raw.var.index = raw_var_names
|
|
690
|
+
|
|
691
|
+
lines = [
|
|
692
|
+
"added",
|
|
693
|
+
f" .layers['{layer_name}'] {'scaled ' if scale_integrated else ''}integrated expression matrix",
|
|
694
|
+
f" .obsp['{mnn_graph_key}'] MNN graph",
|
|
695
|
+
f" .uns['{uns_key}'] integration parameters"
|
|
696
|
+
]
|
|
697
|
+
logg.info(" finished ({time_passed})", deep="\n".join(lines), time=start)
|
|
698
|
+
|
|
699
|
+
# Additional garbage collection #1
|
|
700
|
+
try:
|
|
701
|
+
del (
|
|
702
|
+
seurat_integrated, integrated_matrix, var_names_integrated,
|
|
703
|
+
obs_names_integrated, seurat_obj, seurat_obj_list, features,
|
|
704
|
+
)
|
|
705
|
+
except NameError:
|
|
706
|
+
pass
|
|
707
|
+
|
|
708
|
+
with _RCallbacks(lambda x: None):
|
|
709
|
+
gc.collect()
|
|
710
|
+
ro.r("gc(verbose = FALSE)")
|
|
711
|
+
for i in ["seurat.obj.list", "features", "integration.anchors", "seurat.integrated"]:
|
|
712
|
+
if i in ro.globalenv:
|
|
713
|
+
del ro.globalenv[i]
|
|
714
|
+
gc.collect()
|
|
715
|
+
finally:
|
|
716
|
+
# Additional garbage collection #2
|
|
717
|
+
gc.collect()
|
|
718
|
+
with _RCallbacks(lambda x: None):
|
|
719
|
+
ro.r("gc(verbose = FALSE)")
|
|
720
|
+
gc.collect()
|