clone2vec 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,511 @@
1
+ from __future__ import annotations
2
+
3
+ import scanpy as sc
4
+ import numpy as np
5
+ import pandas as pd
6
+ import scipy.sparse as sp
7
+
8
+ import sys
9
+ from contextlib import nullcontext
10
+
11
+ from tqdm import tqdm
12
+
13
+ logg = sc.logging
14
+
15
+ def _mnn(
16
+ A: pd.DataFrame,
17
+ B: pd.DataFrame,
18
+ k: int = 10,
19
+ metric: str = "euclidean",
20
+ k_trim: int = 3,
21
+ ) -> np.ndarray:
22
+ """
23
+ Find mutual nearest neighbors (MNN) between two datasets A and B.
24
+
25
+ Parameters
26
+ ----------
27
+ A : pd.DataFrame
28
+ First dataset with shape (n_samples_A, n_features).
29
+ B : pd.DataFrame
30
+ Second dataset with shape (n_samples_B, n_features).
31
+ k : int, optional
32
+ Number of nearest neighbors to find, by default 10.
33
+ metric : str, optional
34
+ Distance metric to use, by default "euclidean".
35
+
36
+ Returns
37
+ -------
38
+ np.ndarray
39
+ Array of MNN pairs, where each row is [index_A, index_B] of a MNN pair.
40
+ """
41
+ import pynndescent
42
+
43
+ index1 = pynndescent.NNDescent(A.values, n_neighbors=k, metric=metric)
44
+ neighbors_2to1, _ = index1.query(B.values, k=k)
45
+
46
+ index2 = pynndescent.NNDescent(B.values, n_neighbors=k, metric=metric)
47
+ neighbors_1to2, _ = index2.query(A.values, k=k)
48
+
49
+ rev_pos_2to1 = [{a: p for p, a in enumerate(arr)} for arr in neighbors_2to1]
50
+ rev_pos_1to2 = [{a: p for p, a in enumerate(arr)} for arr in neighbors_1to2]
51
+
52
+ pairs = []
53
+ weights = []
54
+
55
+ for i in range(len(A)):
56
+ js = neighbors_1to2[i]
57
+ cand = []
58
+ scores = []
59
+ wts = []
60
+ for pos, j in enumerate(js):
61
+ rp = rev_pos_2to1[j].get(i)
62
+ if rp is None:
63
+ continue
64
+ rank_sum = pos + rp
65
+ w = 1.0 if k <= 1 else 1.0 - (rank_sum / (2.0 * (k - 1)))
66
+ cand.append(j)
67
+ scores.append(rank_sum)
68
+ wts.append(w)
69
+ if cand:
70
+ order = np.argsort(scores)[: min(k_trim, len(cand))]
71
+ for idx in order:
72
+ pairs.append((A.index[i], B.index[cand[idx]]))
73
+ weights.append(wts[idx])
74
+
75
+ for j in range(len(B)):
76
+ is_ = neighbors_2to1[j]
77
+ cand = []
78
+ scores = []
79
+ wts = []
80
+ for pos, i in enumerate(is_):
81
+ rp = rev_pos_1to2[i].get(j)
82
+ if rp is None:
83
+ continue
84
+ rank_sum = pos + rp
85
+ w = 1.0 if k <= 1 else 1.0 - (rank_sum / (2.0 * (k - 1)))
86
+ cand.append(i)
87
+ scores.append(rank_sum)
88
+ wts.append(w)
89
+ if cand:
90
+ order = np.argsort(scores)[: min(k_trim, len(cand))]
91
+ for idx in order:
92
+ pairs.append((A.index[cand[idx]], B.index[j]))
93
+ weights.append(wts[idx])
94
+
95
+ return np.array(pairs), np.asarray(weights, dtype=float)
96
+
97
+ def _hierarchical_concatenation(
98
+ similarity_matrix: pd.DataFrame,
99
+ method: str = "ward",
100
+ ) -> list[tuple[tuple[str, str], str]]:
101
+ """
102
+ Perform hierarchical concatenation on a similarity matrix.
103
+
104
+ Parameters
105
+ ----------
106
+ similarity_matrix : pd.DataFrame
107
+ DataFrame where rows and columns represent items, and values are similarities.
108
+ method : str, optional
109
+ Linkage method to use for clustering, by default "ward".
110
+
111
+ Returns
112
+ -------
113
+ list[tuple[tuple[str, str], str]]
114
+ List of concatenation steps, each a pair of merged clusters and their new name.
115
+ """
116
+ from scipy.cluster.hierarchy import linkage
117
+ from scipy.spatial.distance import squareform
118
+
119
+ labels = similarity_matrix.columns.tolist()
120
+
121
+ sim_array = similarity_matrix.to_numpy()
122
+ max_similarity = np.max(sim_array)
123
+ dist_array = max_similarity - sim_array
124
+
125
+ condensed_dist = squareform(dist_array, checks=False)
126
+ Z = linkage(condensed_dist, method=method)
127
+
128
+ n = len(labels)
129
+ cluster_names = labels.copy()
130
+ concatenation_steps = []
131
+
132
+ for i, row in enumerate(Z):
133
+ idx1, idx2 = int(row[0]), int(row[1])
134
+ name1 = cluster_names[idx1]
135
+ name2 = cluster_names[idx2]
136
+ new_cluster_name = f"({name1}+{name2})"
137
+ concatenation_steps.append([[name1, name2], new_cluster_name])
138
+ cluster_names.append(new_cluster_name)
139
+
140
+ return concatenation_steps
141
+
142
+ def _affine_transform(
143
+ A: np.ndarray,
144
+ B: np.ndarray,
145
+ order_A: np.ndarray,
146
+ order_B: np.ndarray,
147
+ weights: np.ndarray = None,
148
+ ) -> tuple[np.ndarray, dict]:
149
+ """
150
+ Perform weighted affine least-squares alignment between datasets A and B.
151
+
152
+ Parameters
153
+ ----------
154
+ A : np.ndarray
155
+ First dataset with shape (n_samples_A, n_features) — query.
156
+ B : np.ndarray
157
+ Second dataset with shape (n_samples_B, n_features) — reference.
158
+ order_A : np.ndarray
159
+ Ordering of samples in A to match with B.
160
+ order_B : np.ndarray
161
+ Ordering of samples in B to match with A.
162
+ weights : np.ndarray, optional
163
+ Weights for each sample pair, by default None.
164
+
165
+ Returns
166
+ -------
167
+ tuple[np.ndarray, dict]
168
+ Transformed A and dictionary with rotation matrix, scale, and translation.
169
+ """
170
+ A_matched = A[order_A, :]
171
+ B_matched = B[order_B, :]
172
+
173
+ if weights is not None:
174
+ w_sqrt = np.sqrt(weights)[:, np.newaxis]
175
+ else:
176
+ w_sqrt = np.ones((A_matched.shape[0], 1))
177
+
178
+ ones_col = np.ones((A_matched.shape[0], 1))
179
+ A_aug = np.hstack([A_matched, ones_col])
180
+ A_w = A_aug * w_sqrt
181
+ B_w = B_matched * w_sqrt
182
+
183
+ solution, _, _, _ = np.linalg.lstsq(A_w, B_w, rcond=None)
184
+ M = solution[:-1, :]
185
+ t = solution[-1, :]
186
+
187
+ A_transformed = (A @ M) + t
188
+ transform_info = {
189
+ "matrix": M,
190
+ "translation": t,
191
+ }
192
+
193
+ return A_transformed, transform_info
194
+
195
+ def _write_order(order: list) -> str:
196
+ """
197
+ Function transforms list with order of alignment to string.
198
+
199
+ Parameters
200
+ ----------
201
+ order : list
202
+ List of concatenation steps, each a pair of merged clusters and their new name.
203
+
204
+ Returns
205
+ -------
206
+ str
207
+ String representation of the order of concatenation.
208
+ """
209
+ order_str = ""
210
+ for operation in order:
211
+ order_str += f"[{operation[0][0]}] + [{operation[0][1]}] = [{operation[1]}];"
212
+ return order_str
213
+
214
+ def _read_order(order_str: str) -> list:
215
+ """
216
+ Function transforms string with order of alignment to list.
217
+
218
+ Parameters
219
+ ----------
220
+ order_str : str
221
+ String representation of the order of concatenation.
222
+
223
+ Returns
224
+ -------
225
+ list
226
+ List of concatenation steps, each a pair of merged clusters and their new name.
227
+ """
228
+ order = []
229
+ for operation in order_str.split(";"):
230
+ if operation == "":
231
+ continue
232
+ clusters, new_cluster = operation.split(" = ")
233
+ clusters = clusters.strip("[]").split("] + [")
234
+ order.append([clusters, new_cluster.strip("[]")])
235
+ return order
236
+
237
+ def find_mnn(
238
+ clones: sc.AnnData,
239
+ batch_key: str,
240
+ use_rep: str = "X_pca",
241
+ k: int = 10,
242
+ k_trim: int = 3,
243
+ metric: str = "euclidean",
244
+ uns_key: str = "simple_mnn",
245
+ graph_key: str = "mnn",
246
+ progress_bar: bool = True,
247
+ ) -> None:
248
+ """
249
+ Find anchors between batches in clones.obs[batch_key] using MNNs using clones.obsm[use_rep].
250
+
251
+ Parameters
252
+ ----------
253
+ clones : AnnData
254
+ Annotated data matrix at the clone level.
255
+ batch_key : str
256
+ Column in `clones.obs` containing batch information.
257
+ use_rep : str, optional
258
+ Key in `clones.obsm` containing representation to use for anchor finding, by default "X_pca".
259
+ k : int, optional
260
+ Number of nearest neighbors to use for anchor finding, by default 10.
261
+ metric : str, optional
262
+ Distance metric to use for anchor finding, by default "euclidean".
263
+ uns_key : str, optional
264
+ Key in `clones.uns` to store anchor information, by default "simple_mnn".
265
+ graph_key : str, optional
266
+ Key in `clones.obsp` to store MNN graph, by default "mnn".
267
+ progress_bar : bool, optional
268
+ Whether to show a progress bar, by default True.
269
+
270
+ Returns
271
+ -------
272
+ None
273
+ MNN graph is stored in `clones.obsp[graph_key]`.
274
+ Anchor information is stored in `clones.uns[uns_key]`.
275
+ """
276
+ from itertools import combinations
277
+
278
+ mnn_cols = []
279
+ mnn_rows = []
280
+ mnn_data = []
281
+
282
+ if not pd.api.types.is_string_dtype(clones.obs[batch_key]):
283
+ logg.warning(f"clones.obs['{batch_key}'] is not string. Converting to string.")
284
+ clones.obs[batch_key] = clones.obs[batch_key].astype(str)
285
+
286
+ if not isinstance(clones.obs[batch_key], pd.CategoricalDtype):
287
+ logg.warning(f"clones.obs['{batch_key}'] is not categorical. Converting to categorical.")
288
+ clones.obs[batch_key] = clones.obs[batch_key].astype("category")
289
+
290
+ if not clones.obs_names.is_unique:
291
+ logg.warning("clones.obs_names must be unique. Performing obs_names_make_unique().")
292
+ clones.obs_names_make_unique()
293
+
294
+ try:
295
+ matrix = clones.obsm[use_rep]
296
+ except KeyError:
297
+ raise KeyError(f"clones.obsm['{use_rep}'] is not found. Please provide a valid use_rep.")
298
+
299
+ # Ensure array-like numeric matrix to avoid pandas reindexing introducing NaNs
300
+ # DataFrame inputs are converted to their underlying values for neighbor search
301
+ matrix_values = matrix.values if isinstance(matrix, pd.DataFrame) else matrix
302
+
303
+ line_before = f" using `clones.obsm['{use_rep}']` for neighbor search"
304
+ start = logg.info("finding MNN anchors across batches", deep="\n" + line_before)
305
+
306
+ batches = list(set(clones.obs[batch_key]))
307
+ batches.sort()
308
+
309
+ if sc.settings.verbosity.value >= 2:
310
+ prefix = " "
311
+ progress_bar = True
312
+ else:
313
+ prefix = ""
314
+
315
+ similarities = {}
316
+ pair_indices = list(combinations(range(len(batches)), 2))
317
+ cm = tqdm(
318
+ pair_indices,
319
+ desc=prefix + "pairwise MNNs",
320
+ file=sys.stdout,
321
+ ) if progress_bar else nullcontext(pair_indices)
322
+ with cm as iterator:
323
+ for i, j in iterator:
324
+ pairs, wts = _mnn(
325
+ pd.DataFrame(
326
+ matrix_values[clones.obs[batch_key] == batches[i]],
327
+ index=np.argwhere(clones.obs[batch_key] == batches[i]).flatten(),
328
+ ),
329
+ pd.DataFrame(
330
+ matrix_values[clones.obs[batch_key] == batches[j]],
331
+ index=np.argwhere(clones.obs[batch_key] == batches[j]).flatten(),
332
+ ),
333
+ k=k,
334
+ k_trim=k_trim,
335
+ metric=metric,
336
+ )
337
+
338
+ if pairs.size == 0:
339
+ similarities.setdefault(batches[i], {})
340
+ similarities.setdefault(batches[j], {})
341
+ similarities[batches[i]][batches[j]] = 0
342
+ similarities[batches[j]][batches[i]] = 0
343
+ continue
344
+
345
+ uniq_pairs, inv = np.unique(pairs, axis=0, return_inverse=True)
346
+ uniq_wts = np.zeros(len(uniq_pairs), dtype=float)
347
+ for idx_u, w in zip(inv, wts):
348
+ if w > uniq_wts[idx_u]:
349
+ uniq_wts[idx_u] = w
350
+
351
+ similarities.setdefault(batches[i], {})
352
+ similarities.setdefault(batches[j], {})
353
+
354
+ mnn_cols.extend(uniq_pairs[:, 0])
355
+ mnn_cols.extend(uniq_pairs[:, 1])
356
+
357
+ mnn_rows.extend(uniq_pairs[:, 1])
358
+ mnn_rows.extend(uniq_pairs[:, 0])
359
+
360
+ mnn_data.extend(uniq_wts)
361
+ mnn_data.extend(uniq_wts)
362
+
363
+ similarities[batches[i]][batches[j]] = len(uniq_pairs)
364
+ similarities[batches[j]][batches[i]] = similarities[batches[i]][batches[j]]
365
+
366
+ if graph_key in clones.obsp:
367
+ logg.warning(f"clones.obsp['{graph_key}'] is already found. Overwriting.")
368
+
369
+ # Build a symmetric MNN graph over global indices; ensure full shape matches clones.n_obs
370
+ clones.obsp[graph_key] = sp.csr_matrix(
371
+ (np.asarray(mnn_data, dtype=np.float32), (mnn_cols, mnn_rows)),
372
+ shape=(clones.n_obs, clones.n_obs),
373
+ )
374
+
375
+ similarities = pd.DataFrame(similarities).fillna(0).astype(int)
376
+ integration_order = _hierarchical_concatenation(similarities)
377
+
378
+ # Making sure that there are at least 3 MNNs in each integration step
379
+ for integration_step in integration_order:
380
+ (batch1, batch2), result = integration_step
381
+
382
+ batch1_names = batch1.replace("(", "").replace(")", "").split("+")
383
+ batch2_names = batch2.replace("(", "").replace(")", "").split("+")
384
+
385
+ if similarities.loc[batch1_names, batch2_names].sum().sum() < 3:
386
+ raise ValueError(f"Not enough MNNs between {batch1} and {batch2}. Please increase k.")
387
+
388
+ clones.uns[uns_key] = {
389
+ "batch_key": batch_key,
390
+ "use_rep": use_rep,
391
+ "k": k,
392
+ "k_trim": k_trim,
393
+ "metric": metric,
394
+ "graph_key": graph_key,
395
+ "order": _write_order(integration_order),
396
+ }
397
+ lines = [
398
+ "added",
399
+ f" .obsp['{graph_key}'] symmetric sparse matrix.",
400
+ f" .uns['{uns_key}'] integration order and parameters.",
401
+ ]
402
+ logg.info(" finished ({time_passed})", deep="\n".join(lines), time=start)
403
+
404
+ def _find_order(batch_vector: pd.Series | np.ndarray | list, graph: sp.csr_matrix | np.ndarray):
405
+ """
406
+ Giving the MNN graph, calculating the integration order.
407
+
408
+ Parameters
409
+ ----------
410
+ batch_vector : pd.Series | np.ndarray | list
411
+ Vector of batch labels for each clone.
412
+ graph : sp.csr_matrix | np.ndarray
413
+ Symmetric MNN graph over global indices.
414
+
415
+ Returns
416
+ -------
417
+ list
418
+ Integration order.
419
+ """
420
+ similarities = {}
421
+ batch_categories = list(set(batch_vector))
422
+ for i, batch1 in enumerate(batch_categories):
423
+ if batch1 not in similarities:
424
+ similarities[batch1] = {}
425
+ for j in range(i+1, len(batch_categories)):
426
+ batch2 = batch_categories[j]
427
+ if batch2 not in similarities:
428
+ similarities[batch2] = {}
429
+ similarities[batch1][batch2] = graph[batch_vector == batch1][:, batch_vector == batch2].sum()
430
+ similarities[batch2][batch1] = similarities[batch1][batch2]
431
+
432
+ similarities = pd.DataFrame(similarities).fillna(0)
433
+ integration_order = _hierarchical_concatenation(similarities)
434
+ return _write_order(integration_order)
435
+
436
+ def align(
437
+ clones: sc.AnnData,
438
+ use_rep: str = "clone2vec",
439
+ adj_rep: str = "clone2vec_pa",
440
+ uns_key: str = "simple_mnn",
441
+ weighted: bool = True,
442
+ ) -> None:
443
+ """
444
+ Align clones.obsm[use_rep] using MNNs found in `clones.uns[uns_key]` via procrustes alignment (PA).
445
+
446
+ Parameters
447
+ ----------
448
+ clones : AnnData
449
+ Annotated data matrix at the clone level.
450
+ use_rep : str, optional
451
+ Key in `clones.obsm` containing representation to use for alignment, by default "clone2vec".
452
+ adj_rep : str, optional
453
+ Key in `clones.obsm` to store aligned representation, by default "clone2vec_pa".
454
+ uns_key : str, optional
455
+ Key in `clones.uns` containing anchor information, by default "simple_mnn".
456
+
457
+ Returns
458
+ -------
459
+ None
460
+ Aligned representation is stored in `clones.obsm[adj_rep]`.
461
+ """
462
+ start = logg.info("aligning representation using weighted affine least-squares and MNN anchors")
463
+ try:
464
+ batch_key = clones.uns[uns_key]["batch_key"]
465
+ graph = clones.obsp[clones.uns[uns_key]["graph_key"]]
466
+ if "order" not in clones.uns[uns_key]:
467
+ integration_order = _read_order(_find_order(clones.obs[batch_key].values, graph))
468
+ else:
469
+ integration_order = _read_order(clones.uns[uns_key]["order"])
470
+ except KeyError:
471
+ raise KeyError(f"clones.uns['{uns_key}'] is not full. Please run `find_anchors` first.")
472
+
473
+ try:
474
+ emb = clones.obsm[use_rep].copy()
475
+ except KeyError:
476
+ raise KeyError(f"clones.obsm['{use_rep}'] is not found. Please provide a valid use_rep.")
477
+
478
+ for integration_step in integration_order:
479
+ (batch1, batch2), result = integration_step
480
+
481
+ batch1_samples = batch1.strip("()").split("+")
482
+ batch2_samples = batch2.strip("()").split("+")
483
+
484
+ ref = clones.obs[batch_key].isin(batch1_samples).values
485
+ query = clones.obs[batch_key].isin(batch2_samples).values
486
+
487
+ if sum(ref) < sum(query):
488
+ ref, query = query, ref
489
+
490
+ sub = graph[query][:, ref].tocoo()
491
+ if weighted:
492
+ weights = sub.data
493
+ else:
494
+ weights = None
495
+
496
+ A_aligned, _ = _affine_transform(
497
+ A=emb[query],
498
+ B=emb[ref],
499
+ order_A=sub.row,
500
+ order_B=sub.col,
501
+ weights=weights,
502
+ )
503
+
504
+ emb[query] = A_aligned
505
+
506
+ clones.obsm[adj_rep] = emb
507
+ lines = [
508
+ "added",
509
+ f" .obsm['{adj_rep}'] aligned representation.",
510
+ ]
511
+ logg.info(" finished ({time_passed})", deep="\n".join(lines), time=start)