nova-clone 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. nova_clone/__init__.py +1 -0
  2. nova_clone/algorithm/__init__.py +2 -0
  3. nova_clone/algorithm/_build_intbc_graph.py +135 -0
  4. nova_clone/algorithm/_call_clones_from_coresets.py +235 -0
  5. nova_clone/algorithm/_clone_call_ops.py +258 -0
  6. nova_clone/algorithm/_clone_calling_algorithm.py +1571 -0
  7. nova_clone/algorithm/_compute_cellbc_vs_coreset_overlap_matrix.py +143 -0
  8. nova_clone/algorithm/_compute_coreset_co_assignment_matrix.py +170 -0
  9. nova_clone/algorithm/_find_maximal_coresets.py +428 -0
  10. nova_clone/algorithm/_keep_reliable_edges.py +202 -0
  11. nova_clone/algorithm/_make_coresets_disjoint.py +114 -0
  12. nova_clone/algorithm/_prune_coresets.py +164 -0
  13. nova_clone/algorithm/_prune_custom_intbcs.py +150 -0
  14. nova_clone/algorithm/_prune_likely_homoplastic_intbcs.py +162 -0
  15. nova_clone/algorithm/_prune_likely_sequencing_errors.py +201 -0
  16. nova_clone/algorithm/_prune_low_cellbc_count_intbcs.py +154 -0
  17. nova_clone/algorithm/_prune_low_umi_count_intbcs.py +152 -0
  18. nova_clone/algorithm/_types.py +150 -0
  19. nova_clone/algorithm/_utils.py +53 -0
  20. nova_clone/algorithm/cassiopeia/__init__.py +1 -0
  21. nova_clone/algorithm/cassiopeia/_call_lineage_groups.py +270 -0
  22. nova_clone/caching/__init__.py +24 -0
  23. nova_clone/caching/_cached.py +175 -0
  24. nova_clone/caching/_cached_computation.py +369 -0
  25. nova_clone/caching/_cached_parallel_computation.py +428 -0
  26. nova_clone/caching/_common.py +200 -0
  27. nova_clone/data/__init__.py +1 -0
  28. nova_clone/qc/__init__.py +179 -0
  29. nova_clone/qc/_clone_intbc_sets.py +234 -0
  30. nova_clone/qc/_compare_clone_calls.py +218 -0
  31. nova_clone/qc/_data_retention.py +175 -0
  32. nova_clone/qc/_homoplasy.py +81 -0
  33. nova_clone/qc/_inter_clonal_kinship.py +60 -0
  34. nova_clone/qc/_intra_clonal_kinship.py +172 -0
  35. nova_clone/qc/_multiplet_rate.py +158 -0
  36. nova_clone/qc/_number_of_clones.py +153 -0
  37. nova_clone/qc/_pre_qc.py +485 -0
  38. nova_clone/simulation/HDBSCAN_method_adjusted.py +68 -0
  39. nova_clone/simulation/__init__.py +0 -0
  40. nova_clone/simulation/_caching.py +40 -0
  41. nova_clone/simulation/benchmarking.py +1221 -0
  42. nova_clone/simulation/clone_to_sequences.py +408 -0
  43. nova_clone/simulation/creating_tree.py +416 -0
  44. nova_clone/simulation/evaluation.py +344 -0
  45. nova_clone/simulation/jaccard_hierarchical_by_intBC.py +78 -0
  46. nova_clone/simulation/plot_benchmark_lines.py +226 -0
  47. nova_clone/simulation/slice_benchmark.py +640 -0
  48. nova_clone/simulation/slice_benchmark_clone_calling.py +68 -0
  49. nova_clone-0.1.0.dist-info/METADATA +298 -0
  50. nova_clone-0.1.0.dist-info/RECORD +53 -0
  51. nova_clone-0.1.0.dist-info/WHEEL +5 -0
  52. nova_clone-0.1.0.dist-info/licenses/LICENSE +28 -0
  53. nova_clone-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1571 @@
1
+ """
2
+ Module for the clone calling algorithm.
3
+
4
+ The main function exposed by this module is the `clone_calling_algorithm`.
5
+ """
6
+
7
+ import logging
8
+ import sys
9
+ import time
10
+ from collections import defaultdict
11
+ from copy import deepcopy
12
+ from typing import Any, Callable, Dict, List, Optional
13
+
14
+ import numpy as np
15
+ import pandas as pd
16
+ import seaborn as sns
17
+
18
+ from ._build_intbc_graph import build_intbc_graph
19
+ from ._call_clones_from_coresets import call_clones_from_coresets
20
+ from ._compute_coreset_co_assignment_matrix import compute_coreset_co_assignment_matrix
21
+ from ._find_maximal_coresets import find_maximal_coresets
22
+ from ._keep_reliable_edges import keep_reliable_edges
23
+ from ._make_coresets_disjoint import *
24
+ from ._prune_coresets import prune_coresets
25
+ from ._prune_custom_intbcs import prune_custom_intbcs
26
+ from ._prune_likely_homoplastic_intbcs import prune_likely_homoplastic_intbcs
27
+ from ._prune_likely_sequencing_errors import prune_likely_sequencing_errors
28
+ from ._prune_low_cellbc_count_intbcs import prune_low_cellbc_count_intbcs
29
+ from ._prune_low_umi_count_intbcs import prune_low_umi_count_intbcs
30
+ from ._types import *
31
+ from ._utils import _cell_intbc_sets_from_intbc_tables
32
+
33
+
34
+ def _init_logger():
35
+ logger = logging.getLogger(__name__)
36
+ logger.setLevel(logging.INFO)
37
+ fmt_str = "[%(asctime)s] - %(name)s - %(levelname)s - %(message)s"
38
+ formatter = logging.Formatter(fmt_str)
39
+
40
+ consoleHandler = logging.StreamHandler(sys.stdout)
41
+ consoleHandler.setFormatter(formatter)
42
+ logger.addHandler(consoleHandler)
43
+
44
+
45
+ _init_logger()
46
+
47
+
48
+ class CloneCallingAlgorithmReturnType(NamedTuple):
49
+ """
50
+ Object returned by `clone_calling_algorithm`
51
+
52
+ Attributes:
53
+ clone_call: The clone call
54
+ recursive_clone_call: The recursive clone call.
55
+ recursive_clone_call_integration_hierarchy: Describes the subclonal
56
+ structure of each clone in a succint way, and hence allows easy
57
+ parsing of the recursive_clone_call. E.g.:
58
+ {
59
+ "clone_0": ({"AAA", "BBB"},
60
+ {
61
+ "clone_0": ({"CCC"},
62
+ None),
63
+ "clone_1": ({"DDD"},
64
+ None),
65
+ }),
66
+ }
67
+ metadata: General metadata for debugging etc.
68
+ total_time: Total time taken
69
+ """
70
+
71
+ clone_call: CloneCallType
72
+ recursive_clone_call: Dict[CloneType, "CloneCallingAlgorithmReturnType"]
73
+ recursive_clone_call_integration_hierarchy: Any = None
74
+ total_time: Optional[float] = None
75
+ metadata: Any = None
76
+
77
+
78
+ def _is_connected(intbc_graph: GraphType) -> bool:
79
+ """
80
+ Returns True if the graph is connected, False otherwise.
81
+ """
82
+ if len(intbc_graph[1]) == 0:
83
+ return True
84
+ edge_counts, node_counts = intbc_graph
85
+ visited = {node: False for node in node_counts}
86
+
87
+ def _dfs(node):
88
+ visited[node] = True
89
+ for neighbor in edge_counts[node]:
90
+ if not visited[neighbor]:
91
+ _dfs(neighbor)
92
+
93
+ _dfs(next(iter(node_counts)))
94
+ return all(visited.values())
95
+
96
+
97
+ def test__is_connected():
98
+ intbc_graph = (
99
+ {
100
+ "CCC": {"AAA": 5, "BBB": 5},
101
+ "AAA": {"CCC": 5, "BBB": 10, "DDD": 5},
102
+ "BBB": {"CCC": 5, "AAA": 10, "DDD": 5},
103
+ "DDD": {"AAA": 5, "BBB": 5},
104
+ },
105
+ {"CCC": 5, "AAA": 10, "BBB": 10, "DDD": 5},
106
+ )
107
+ assert _is_connected(intbc_graph)
108
+
109
+
110
+ def test__empty_is_connected():
111
+ intbc_graph = ({}, {})
112
+ assert _is_connected(intbc_graph)
113
+
114
+
115
+ def test__is_connected_2():
116
+ intbc_graph = ({"CCC": {}, "DDD": {}}, {"CCC": 5, "DDD": 5})
117
+ assert not _is_connected(intbc_graph)
118
+
119
+
120
+ def _pop_highest_frequency_vertex(
121
+ intbc_graph: GraphType,
122
+ ) -> Tuple[IntBCType, GraphType]:
123
+ """
124
+ Pop the highest frequency vertex from the graph.
125
+ """
126
+ edge_counts, node_counts = intbc_graph
127
+ if not node_counts:
128
+ raise ValueError("The graph is empty.")
129
+ highest_frequency_vertex = max(node_counts, key=node_counts.get)
130
+ new_node_counts = {
131
+ node: count
132
+ for node, count in node_counts.items()
133
+ if node != highest_frequency_vertex
134
+ }
135
+ new_edge_counts = {
136
+ node: {
137
+ neighbor: count
138
+ for neighbor, count in neighbors.items()
139
+ if neighbor != highest_frequency_vertex
140
+ }
141
+ for node, neighbors in edge_counts.items()
142
+ if node != highest_frequency_vertex
143
+ }
144
+ return highest_frequency_vertex, (new_edge_counts, new_node_counts)
145
+
146
+
147
+ def test__pop_highest_frequency_vertex():
148
+ intbc_graph = (
149
+ {
150
+ "CCC": {"AAA": 5, "BBB": 5},
151
+ "AAA": {"CCC": 5, "BBB": 10, "DDD": 5},
152
+ "BBB": {"CCC": 5, "AAA": 10, "DDD": 5},
153
+ "DDD": {"AAA": 5, "BBB": 5},
154
+ },
155
+ {"CCC": 5, "AAA": 11, "BBB": 10, "DDD": 6},
156
+ )
157
+ intbc, new_intbc_graph = _pop_highest_frequency_vertex(intbc_graph)
158
+ assert new_intbc_graph == (
159
+ {"CCC": {"BBB": 5}, "BBB": {"CCC": 5, "DDD": 5}, "DDD": {"BBB": 5}},
160
+ {"CCC": 5, "BBB": 10, "DDD": 6},
161
+ )
162
+ assert intbc == "AAA"
163
+ intbc, new_intbc_graph = _pop_highest_frequency_vertex(new_intbc_graph)
164
+ assert new_intbc_graph == ({"CCC": {}, "DDD": {}}, {"CCC": 5, "DDD": 6})
165
+ assert intbc == "BBB"
166
+ intbc, new_intbc_graph = _pop_highest_frequency_vertex(new_intbc_graph)
167
+ assert new_intbc_graph == ({"CCC": {}}, {"CCC": 5})
168
+ assert intbc == "DDD"
169
+ intbc, new_intbc_graph = _pop_highest_frequency_vertex(new_intbc_graph)
170
+ assert new_intbc_graph == ({}, {})
171
+ assert intbc == "CCC"
172
+
173
+
174
+ def test__pop_highest_frequency_vertex_raises():
175
+ intbc_graph = ({}, {})
176
+ try:
177
+ _pop_highest_frequency_vertex(intbc_graph)
178
+ assert False
179
+ except ValueError:
180
+ assert True
181
+
182
+
183
+ def _is_empty(intbc_table_per_sample_for_clone):
184
+ total_numer_of_rows = sum(
185
+ intbc_table.shape[0]
186
+ for intbc_table in intbc_table_per_sample_for_clone.values()
187
+ )
188
+ return total_numer_of_rows == 0
189
+
190
+
191
+ def remove_empty_clones(clone_call: CloneCallType) -> CloneCallType:
192
+ """
193
+ Remove empty clones from the clone call.
194
+
195
+ A clone is considered empty if it has no cells assigned to it, or if
196
+ its intBC set is empty.
197
+ """
198
+ non_empty_clones = {
199
+ clone
200
+ for clone, cells in clone_call.clone_to_cells_mapping.items()
201
+ if len(cells) > 0 and len(clone_call.clone_intbc_sets.get(clone, set())) > 0
202
+ }
203
+
204
+ clone_to_cells_mapping = {
205
+ clone: cells
206
+ for clone, cells in clone_call.clone_to_cells_mapping.items()
207
+ if clone in non_empty_clones
208
+ }
209
+ clone_intbc_sets = {
210
+ clone: intbcs
211
+ for clone, intbcs in clone_call.clone_intbc_sets.items()
212
+ if clone in non_empty_clones
213
+ }
214
+ cell_to_clones_mapping = {
215
+ cellbc: {clone for clone in clones if clone in non_empty_clones}
216
+ for cellbc, clones in clone_call.cell_to_clones_mapping.items()
217
+ }
218
+
219
+ return CloneCallType(
220
+ clone_to_cells_mapping=clone_to_cells_mapping,
221
+ cell_to_clones_mapping=cell_to_clones_mapping,
222
+ clone_intbc_sets=clone_intbc_sets,
223
+ cell_intbc_sets=clone_call.cell_intbc_sets,
224
+ )
225
+
226
+
227
+ def test_remove_empty_clones():
228
+ clone_call = CloneCallType(
229
+ clone_to_cells_mapping={
230
+ "clone_0": {"c1"},
231
+ "clone_1": set(),
232
+ "clone_2": {"c2"},
233
+ "clone_3": {"c4"},
234
+ },
235
+ cell_to_clones_mapping={
236
+ "c1": {"clone_0"},
237
+ "c2": {"clone_2"},
238
+ "c3": {"clone_1", "clone_2"},
239
+ "c4": {"clone_3"},
240
+ },
241
+ clone_intbc_sets={
242
+ "clone_0": {"A"},
243
+ "clone_1": set(),
244
+ "clone_2": {"C"},
245
+ "clone_3": set(),
246
+ },
247
+ cell_intbc_sets={"c1": {"A"}, "c2": {"C"}, "c3": {"C"}, "c4": set()},
248
+ )
249
+ out = remove_empty_clones(clone_call)
250
+ assert set(out.clone_to_cells_mapping.keys()) == {"clone_0", "clone_2"}
251
+ assert set(out.clone_intbc_sets.keys()) == {"clone_0", "clone_2"}
252
+ assert out.cell_to_clones_mapping["c3"] == {"clone_2"}
253
+ assert out.cell_to_clones_mapping["c4"] == set()
254
+
255
+
256
+ def clone_calling_algorithm(
257
+ ##### Public parameters (defaults should work well on most datasets) #####
258
+ # Phase 1 parameters
259
+ intbc_table_per_sample: Dict[str, pd.DataFrame],
260
+ min_intbc_umi_count: int = 10,
261
+ custom_intbc_filtering_function: Callable[[str], bool] = lambda intbc: False,
262
+ min_intbc_cellbc_count: int = 10,
263
+ edit_distance_cutoff: int = 4,
264
+ # Phase 2 parameters
265
+ min_cells_assigned: int = 10,
266
+ min_intbc_fraction: float = 0.05,
267
+ edit_distance_cutoff_phase_2: Optional[int] = None, # If `None`, defaults to `edit_distance_cutoff` value
268
+ # Recursive step parameters
269
+ recursive: bool = False,
270
+ recursive_min_fraction_uniquely_mapped_cells_to_recurse: float = 0.7,
271
+ recursive_min_cells_assigned: int = 10,
272
+ # Single-intBC clone recovery
273
+ recover_single_intbc_clones: bool = False,
274
+ ##### Private parameters (we do not expect the user to ever have to change these) #####
275
+ _sequencing_error_pruning_step__co_occurrence_cutoff: float = 0.5,
276
+ _homoplastic_pruning_step__p_max_cutoff: float = 0.5,
277
+ _reliable_edges_step__ratio_cutoff: float = 0.5,
278
+ _preliminary_coreset_building_step__min_coreset_size: int = 2, # Private parameter, not exposed to the user. Used for the recursive call of the algorithm.
279
+ _preliminary_coreset_building_step__min_overlap_for_merging_coresets: int = 2, # Private parameter, not exposed to the user. Used for the recursive call of the algorithm.
280
+ _coreset_dealiasing_step__min_overlap_for_assigning_cellbc: int = 2, # Private parameter, not exposed to the user. Used for the recursive call of the algorithm.
281
+ _coreset_dealiasing_step__co_occurrence_cutoff: float = 0.5, # Private parameter, not exposed to the user. Used for the recursive call of the algorithm.
282
+ _coreset_disjointing_step__make_disjoint: bool = True, # Private parameter, not exposed to the user. Used for testing.
283
+ _final_cell_assignment_step__min_overlap_for_assigning_cellbc: int = 1, # Private parameter, not exposed to the user. Used for the recursive call of the algorithm.
284
+ _call_clones_from_coresets_step__min_intbc_counts_for_adding_intbc_to_clone_intbc_set: Optional[int] = None, # Private parameter, not exposed to the user. Used for testing.
285
+ ) -> CloneCallingAlgorithmReturnType:
286
+ """
287
+ Args:
288
+ intbc_table_per_sample: Each value should be a pandas DataFrame with the following columns:
289
+ "cellBC" (str), "intBC" (str), "UMI_count" (int). For example, a row might be:
290
+ ["AAACCCAAGAGCCTGA", "ACCTCATGATATTC", 45]
291
+ indicating that the intBC "ACCTCATGATATTC" was observed 45 times in the cellBC "AAACCCAAGAGCCTGA".
292
+ There should be one intBC table per sample; the sample name acts as the key of the dictionary.
293
+ min_intbc_umi_count: IntBCs with less than this number of UMI_counts will be discarded by the algorithm.
294
+ The default value is 10, which is effective at removing sequencing errors and background RNA and
295
+ sequencing errors. For some datasets, such as the macsGESTALT which we analyze, we lower this
296
+ threshold to two UMIs, as described in the main text. More generally, we recommend to set the UMI
297
+ cutoff by considering a histogram of UMI counts across all (cellBC, intBC) pairs (rows in the input
298
+ table) and use the leftmost mode (in case the count distribution is multi-modal) or a low quantile
299
+ (otherwise).
300
+ custom_intbc_filtering_function: IntBCs for which this function gives
301
+ `True` will be discarded.
302
+ min_intbc_cellbc_count: IntBCs appearing in less than this number of
303
+ cellBCs will be discarded.
304
+ edit_distance_cutoff: Edit distance cutoff. E.g. if you expect there to
305
+ exist sequencing errors at edit distance 4 but not 5, please set this to 4.
306
+ To be precise about how this is used: When building clonal signatures (phase 1 of
307
+ the algorithm), we identify putative sequencing errors by checking for
308
+ intBCs `x` frequently co-occurring with another (more frequent) intBC
309
+ `y` at an edit distance <= edit_distance_cutoff. If so, we remove `x`.
310
+ Similarly, in the second phase of the algorithm, when we reconstruct
311
+ the intBC set for each clone de novo, we discard each intBC with an
312
+ edit distance <= edit_distance_cutoff to some more abundant intBC already
313
+ kept in the clone's intBC set.
314
+ We allow setting the cutoff for phase 2 differently with `edit_distance_cutoff_phase_2`.
315
+ min_cells_assigned: Coresets with less than this number of mapped cells
316
+ will be dropped. If you wish to recover smaller clones, you can lower
317
+ this, however, note that since all \intBCs are supported by at least
318
+ min_intbc_cellbc_count cellBCs (by default 10, and we do not recommend
319
+ changing this), all clones will have at least 10 mapped cellBCs.
320
+ min_intbc_fraction:
321
+ Following up on the above argument, we can also specify the
322
+ minimum percentage of cells in the clone containing the intBC
323
+ needed to include the intBC in the clone's intBC set. We set this
324
+ to 5% by default to recover the intBC tail too. Set this to e.g.
325
+ 20% if you only wish to retain only high-frequency intBCs.
326
+ edit_distance_cutoff_phase_2: See description of parameter edit_distance_cutoff.
327
+ If `None` (default) this is set equal to edit_distance_cutoff. This should work
328
+ well in most cases. Reasons for setting this value to something different than
329
+ `edit_distance_cutoff` are (i) increase retention of intBCs after clone calling;
330
+ note that while removing sequencing errors during the first phase (finding clonal
331
+ signatures) is critical (or else we might overmerge clones), when building intBC
332
+ sets de novo is it less critical because in the worst case, sequencing errors
333
+ lead to "duplicate" characters which will likely get dropped due to low frequency
334
+ via `min_intbc_fraction` anyway, and (ii) the user might want to curate the intBCs
335
+ in each clone themselves, in which case they can set edit_distance_cutoff_phase_2=0
336
+ to turn off this second sequencing error filter.
337
+ recursive: If True, then we will recursively call the algorithm on each
338
+ clone, thus resolving the subclonal structure derived by the
339
+ hierarchy of integrations. When using this option, we recommend
340
+ (and will enforce) using:
341
+ min_intbc_fraction <= 0.01
342
+ recursive_min_fraction_uniquely_mapped_cells_to_recurse: If recursive
343
+ is True, then this determines the min fraction of cells that have
344
+ to be uniquely mapped to a clone's subclones before we accept the
345
+ subclonal decomposition of that clone. This thus serves as a
346
+ stopping criteria. For example, a clone might have two resolvable
347
+ subclones comprising 35% and 31% of the clone's cells respectively,
348
+ with the remaining 34% unmapped (i.e. discarded). Hence, if this
349
+ argument is set to 0.7 (i.e. 70%), then we would NOT accept this
350
+ recursive decomposition and just retain the original clone. Instead,
351
+ if the subclones comprised 35% and 35% of the clone's cells, then
352
+ we would exactly meet the 70% threshold and accept the recursive
353
+ decomposition.
354
+ recursive_min_cells_assigned: If recursive
355
+ is True, a lower bound on the size of the subclones we want to find.
356
+ Subclones tend to be smaller than the parent clone, so it is a good
357
+ idea to set this lower than the original
358
+ min_cells_assigned.
359
+ recover_single_intbc_clones: If `True`, single-intBC clones will also be
360
+ recovered. Use this if your data contains single-intBC clones. There
361
+ is an inherent tradeoff in trying to recover single-intBC clones: since
362
+ these clones have weaker statistical support, unless you are certain
363
+ that your data has negligible levels of homoplasy, you might end up
364
+ accidentally overmerging clones. If evolving tracer marks are
365
+ available, we recommend using these to confirm that the single-intBC
366
+ clones are high quality, and if there is overmerging we recommend
367
+ splitting into different clones based on evolving lineage marks.
368
+ _sequencing_error_pruning_step__co_occurrence_cutoff: This establishes
369
+ how frequently intBC `x` needs to co-occur with intBC `y` having a
370
+ similar sequence (as determined by the edit_distance_cutoff above)
371
+ to be considered a sequencing error. This frequency cutoff is as
372
+ follows: p(y | x) >= co_occurrence_cutoff. Thus, using a smaller
373
+ value for this argument detects more sequencing errors but also
374
+ risks discarding intBCs which are not actually sequencing errors
375
+ (i.e. higher recall, lower precision). The default value is 0.5;
376
+ we do not recommend the user to change this value, hence we have
377
+ made it a private parameter.
378
+ _homoplastic_pruning_step__p_max_cutoff: This determines how homoplastic
379
+ intBCs are found. Homoplastic intBCs appear in multiple clones and
380
+ therefore may not co-occurr with any other intBC at high
381
+ probability. This p_max_cutoff argument specifies that we consider
382
+ an intBC `x` to be homoplastic if p_max_cutoff > max_y p(y | x).
383
+ Thus, setting this argument to 0 bypasses this step. The default
384
+ value is 0.5; we do not recommend the user to change this value,
385
+ hence we have made it a private parameter.
386
+ _reliable_edges_step__ratio_cutoff: We say that an edge in the intBC
387
+ graph is "reliable" if we are very certain that there exists one
388
+ and only one clone in which its two intBCs are integrations.
389
+ Since the graph's been pruned from sequencing errors at this point,
390
+ reliable pairs will have a large ratio. Here the ratio is defined
391
+ as: ratio(x, y) = c(x, y) / min(c_x, c_y) = max { p(x|y), p(y|x) }
392
+ The default value is 0.5; we do not recommend the user to change
393
+ this value, hence we have made it a private parameter.
394
+ _preliminary_coreset_building_step__min_coreset_size: The minimum size
395
+ of an intBC set, all of which are pairwise reliably co-occurring,
396
+ which implies that said set of intBCs exists in one and only one
397
+ clone. Such sets are called "coresets". This is a private parameter
398
+ not exposed to the user. Please do not use it. The only reason it is
399
+ here is because it is relaxed in the recursive call of the algorithm.
400
+ _preliminary_coreset_building_step__min_overlap_for_merging_coresets:
401
+ Coresets also have the property that if two coresets A and B
402
+ intersect in `min_overlap_for_merging_coresets`, then they
403
+ represent the same clone and their union is a coreset for that
404
+ clone. This is a private parameter not exposed to the user. Please
405
+ do not use it. The only reason it is here is because it is relaxed in
406
+ the recursive call of the algorithm.
407
+ _coreset_dealiasing_step__min_overlap_for_assigning_cell: After we
408
+ compute the preliminary coresets, it is possible that two of these
409
+ refer to the same clone. Because of this, we map cellBCs to
410
+ preliminary coresets to determine which coresets are aliased. To
411
+ map a cellBC to a coreset, we require that this number of intBCs
412
+ overlap with the coreset. This is a private parameter not exposed
413
+ to the user. Please do not use it. The only reason it is here is
414
+ because it is relaxed in the recursive call of the algorithm.
415
+ _coreset_dealiasing_step__co_occurrence_cutoff: To identify aliased
416
+ coresets (i.e. representing the same clone), we look at the cellBCs
417
+ mapped to these two coresets. Let m_1 and m_2 be the number of
418
+ cellBCs mapped to each, and m_12 the number of cellBCs mapped to
419
+ both. If m_12 / min(m_1, m_2) >= co_occurrence_cutoff (and if
420
+ min(m_1, m_2) >= min_cells_assigned below) we consider the coresets
421
+ to be aliased. Of course, the co_occurrence_cutoff and
422
+ min_cells_assigned should be chosen to exclude multiplets as the
423
+ reason for co-occurrence. This is a private parameter not exposed
424
+ to the user. Please do not use it. The only reason it is here is
425
+ for testing purposes.
426
+ _coreset_disjointing_step__make_disjoint: Once all coresets have been
427
+ found (which map one-to-one with clones), we can remove homoplastic
428
+ intBCs from the coresets to create an even stronger signature for
429
+ mapping cellBCs. This is a private parameter not exposed to the user. Please
430
+ do not use it. The only reason it is here is for testing purposes.
431
+ _final_cell_assignment_step__min_overlap_for_assigning_cellbc: To
432
+ map a cellBC to a coreset, we require that this number of intBCs
433
+ overlap with the coreset. This is a private parameter not exposed
434
+ to the user. Please do not use it. The only reason it is here is
435
+ because it is relaxed in the recursive call of the algorithm.
436
+ _call_clones_from_coresets_step__min_intbc_counts_for_adding_intbc_to_clone_intbc_set:
437
+ After mapping all the cells to that clone, we can calculate the
438
+ *full* intBC set for each clone (which goes beyond the coreset,
439
+ i.e. it includes homoplastic intBCs). This argument specifies the
440
+ minimum number of cells in that clone containing the intBC which
441
+ are needed to include the intBC in that clone's final intBC set.
442
+ If `None` (default), this is set to `min_intbc_cellbc_count`.
443
+ This is a private parameter not exposed to the user. Please do not
444
+ use it. The only reason it is here is for testing purposes.
445
+
446
+ Returns:
447
+ A `results` NamedTuple of type CloneCallingAlgorithmReturnType, which contains
448
+ the clone call in the field `clone_call`. The `clone_call` itself is also a
449
+ NamedTuple; the most important fields are:
450
+ - `results.clone_call.cell_to_clones_mapping (Dict[str, Set[str]])`:
451
+ Dictionary mapping cellBCs to clone IDs. A cellBCs mapping to more than one
452
+ clone is a multiplet, and a cellBC mapping to no clone is an empty cellBC or
453
+ a dropped cellBC (e.g. due to low quality).
454
+ - `results.clone_call.clone_to_cells_mapping (Dict[str, Set[str]])`:
455
+ Dictionary clone IDs to cellBCs.
456
+ - `results.clone_call.clone_intbc_sets (Dict[str, Set[str]])`:
457
+ Dictionary mapping clone IDs to its set of IntBCs.
458
+ - `results.clone_call.cell_intbc_sets (Dict[str, Set[str]])`:
459
+ Dictionary mapping cellBCs to its set of IntBCs.
460
+ NOTE: the cellBCs are prefixed with the sample name in the `results` (to avoid
461
+ cellBC collision between different samples.)
462
+ """
463
+ if recursive:
464
+ if min_intbc_fraction > 0.05:
465
+ raise ValueError("When using recursive mode, we recommend (and will enforce) using: min_intbc_fraction <= 0.05. Otherwise, the subclonal intBCs will be discarded and little recursive structure will be found.")
466
+
467
+ if edit_distance_cutoff_phase_2 is None:
468
+ edit_distance_cutoff_phase_2 = edit_distance_cutoff
469
+
470
+ if _call_clones_from_coresets_step__min_intbc_counts_for_adding_intbc_to_clone_intbc_set is None:
471
+ _call_clones_from_coresets_step__min_intbc_counts_for_adding_intbc_to_clone_intbc_set = min_intbc_cellbc_count
472
+
473
+ if recover_single_intbc_clones:
474
+ # We set the min_coreset_size to size to 1.
475
+ # This retains isolated vertices in the starting coreset family.
476
+ _preliminary_coreset_building_step__min_coreset_size = 1
477
+ # Also, if the user is running in single-intBC mode, the homoplasy pruning step should be
478
+ # skipped, because it discards single-intBC clones. The user has acknowledged
479
+ # the limitations of recovering clones with weaker statistical support, and will
480
+ # check the post-QC plots closely.
481
+ _homoplastic_pruning_step__p_max_cutoff = -1
482
+
483
+
484
+ st = time.time()
485
+
486
+ logger = logging.getLogger(__name__)
487
+ print(
488
+ "*********************************************************************************"
489
+ )
490
+ print(
491
+ '***** Phase 1 of algorithm: Construct clonal signatures (a.k.a. "coresets") *****'
492
+ )
493
+ print(
494
+ "*********************************************************************************"
495
+ )
496
+
497
+ # Validate arguments
498
+ validate_intbc_table_per_sample(intbc_table_per_sample=intbc_table_per_sample)
499
+ original_intbc_table_per_sample = deepcopy(
500
+ intbc_table_per_sample
501
+ ) # After identifying the coresets, we will use the original intBC tables to map cells to clones, call sequencing errors and homoplastic intBCs.
502
+ for sample in intbc_table_per_sample:
503
+ if "_" in sample:
504
+ raise ValueError("The sample names should not contain underscores.")
505
+
506
+ # Filter intBCs based on min_intbc_umi_count
507
+ intbc_table_per_sample = prune_low_umi_count_intbcs(
508
+ intbc_table_per_sample=intbc_table_per_sample,
509
+ min_intbc_umi_count=min_intbc_umi_count,
510
+ ).intbc_table_per_sample
511
+
512
+ # Filter intBCs based on `custom_intbc_filtering_function`
513
+ intbc_table_per_sample = prune_custom_intbcs(
514
+ intbc_table_per_sample=intbc_table_per_sample,
515
+ custom_intbc_filtering_function=custom_intbc_filtering_function,
516
+ ).intbc_table_per_sample
517
+
518
+ # Filter intBCs based on `min_intbc_cellbc_count`
519
+ intbc_table_per_sample = prune_low_cellbc_count_intbcs(
520
+ intbc_table_per_sample=intbc_table_per_sample,
521
+ min_intbc_cellbc_count=min_intbc_cellbc_count,
522
+ ).intbc_table_per_sample
523
+
524
+ # Build the intBC graph.
525
+ intbc_graph = build_intbc_graph(
526
+ intbc_table_per_sample=intbc_table_per_sample,
527
+ ).intbc_graph
528
+
529
+ # Prune intBCs with small edit distance and high co-occurrence to some more
530
+ # frequent intBC.
531
+ # This gets rid of many (high-count) sequencing errors.
532
+ prune_likely_sequencing_errors_return = prune_likely_sequencing_errors(
533
+ intbc_graph=intbc_graph,
534
+ edit_distance_cutoff=edit_distance_cutoff, # noqa
535
+ co_occurrence_cutoff=_sequencing_error_pruning_step__co_occurrence_cutoff, # noqa
536
+ )
537
+ intbc_graph = prune_likely_sequencing_errors_return.intbc_graph
538
+ pruned_sequencing_errors = (
539
+ prune_likely_sequencing_errors_return.sequencing_errors_and_originals
540
+ )
541
+
542
+ # Prune intBCs with p_max too low.
543
+ # This gets rid of highly homoplastic intBCs.
544
+ intbc_graph = prune_likely_homoplastic_intbcs(
545
+ intbc_graph=intbc_graph,
546
+ p_max_cutoff=_homoplastic_pruning_step__p_max_cutoff,
547
+ ).intbc_graph
548
+
549
+ # Keep reliable edges only
550
+ intbc_graph = keep_reliable_edges(
551
+ intbc_graph=intbc_graph,
552
+ ratio_cutoff=_reliable_edges_step__ratio_cutoff,
553
+ ).intbc_graph
554
+
555
+ # Infer preliminary coresets by merging cliques
556
+ preliminary_coresets = find_maximal_coresets(
557
+ graph=intbc_graph,
558
+ min_coreset_size=_preliminary_coreset_building_step__min_coreset_size, # noqa
559
+ min_overlap_for_merging_coresets=_preliminary_coreset_building_step__min_overlap_for_merging_coresets, # noqa
560
+ recover_single_intbc_clones=recover_single_intbc_clones,
561
+ ).maximal_coresets
562
+
563
+ # Find aliased coresets; we first need to assign the cells to the coresets.
564
+ logger.info("Going to compute_coreset_co_assignment_matrix ...")
565
+ cellbc_intbc_sets = _cell_intbc_sets_from_intbc_tables(
566
+ intbc_table_per_sample=intbc_table_per_sample,
567
+ )
568
+ preliminary_coreset_co_assignment_matrix = compute_coreset_co_assignment_matrix(
569
+ cellbc_intbc_sets=cellbc_intbc_sets,
570
+ coresets=preliminary_coresets,
571
+ min_overlap_for_assigning_cellbc=_coreset_dealiasing_step__min_overlap_for_assigning_cellbc,
572
+ ).coreset_co_assignment_matrix
573
+
574
+ # Prune preliminary coresets that are aliased (so we end with one per clone)
575
+ pruned_coresets = prune_coresets(
576
+ coresets=preliminary_coresets,
577
+ coreset_co_assignment_matrix=preliminary_coreset_co_assignment_matrix,
578
+ co_occurrence_cutoff=_coreset_dealiasing_step__co_occurrence_cutoff,
579
+ min_cells_assigned=min_cells_assigned,
580
+ ).coresets
581
+
582
+ if _coreset_disjointing_step__make_disjoint:
583
+ disjoint_coresets = make_coresets_disjoint(
584
+ coresets=pruned_coresets,
585
+ ).coresets
586
+ else:
587
+ disjoint_coresets = pruned_coresets
588
+
589
+ print("********************************************************************")
590
+ print("***** Phase 2 of algorithm: Call clones from clonal signatures *****")
591
+ print("********************************************************************")
592
+
593
+ call_clones_from_coresets_return = call_clones_from_coresets(
594
+ intbc_table_per_sample=original_intbc_table_per_sample,
595
+ min_intbc_umi_count=min_intbc_umi_count,
596
+ coresets=disjoint_coresets,
597
+ min_overlap_for_assigning_cellbc=_final_cell_assignment_step__min_overlap_for_assigning_cellbc,
598
+ min_intbc_counts_for_adding_intbc_to_clone_intbc_set=_call_clones_from_coresets_step__min_intbc_counts_for_adding_intbc_to_clone_intbc_set,
599
+ min_intbc_fraction_for_adding_to_clone_intbc_set=min_intbc_fraction,
600
+ edit_distance_cutoff=edit_distance_cutoff_phase_2,
601
+ custom_intbc_filtering_function=custom_intbc_filtering_function,
602
+ )
603
+ clone_call = call_clones_from_coresets_return.clone_call
604
+
605
+ # Remove empty clones -- can happen in very extreme cases (low quality input data)
606
+ # due to e.g. (1) the disjointing step, which causes a putative clonal signature
607
+ # to drop from 2 intBCs to 1, leading to 0 cells assigned in phase 2, as well as
608
+ # (2) the intBC quality filter in step 2 rejects even the intBCs in the clonal
609
+ # signature, leading to a clone with empty intBC set.
610
+ clone_call = remove_empty_clones(clone_call)
611
+
612
+ # Call recursively on each clone, if requested.
613
+ if recursive:
614
+ recursive_clone_call = {}
615
+ recursive_clone_call_integration_hierarchy = {}
616
+ clones = list(clone_call.clone_to_cells_mapping.keys())
617
+ for clone in clones:
618
+ print(f"Recursively calling clones on clone: {clone} ...")
619
+
620
+ cells_in_clone = sorted(list(clone_call.clone_to_cells_mapping[clone]))
621
+ # A clone may span multiple samples
622
+ cells_in_clone_per_sample = defaultdict(set)
623
+ for cell in cells_in_clone:
624
+ sample, cellbc = cell.split("_", 1)
625
+ cells_in_clone_per_sample[sample].add(cellbc)
626
+
627
+ # Since we are going to be popping intBCs until the graph disconnects,
628
+ # we better start with a connected graph. This is ensured by the coresets.
629
+ # Specifically, we consider the pruned coresets (which are dealiased), and
630
+ # not the disjointed ones (since the disjointed ones lose homoplastic intBCs).
631
+ clone_id = int(clone[len("clone_") :])
632
+ pruned_coreset_for_clone = pruned_coresets[clone_id]
633
+ disjoint_coreset_for_clone = disjoint_coresets[clone_id]
634
+ assert all(
635
+ [
636
+ intbc in pruned_coreset_for_clone
637
+ for intbc in disjoint_coreset_for_clone
638
+ ]
639
+ )
640
+ clone_intbc_set = clone_call.clone_intbc_sets[clone]
641
+ del disjoint_coreset_for_clone
642
+ intbc_set_to_disconnect = set(
643
+ [x for x in pruned_coreset_for_clone if x in clone_intbc_set]
644
+ )
645
+
646
+ # Phase 1: Identify the progenitor intBC set.
647
+ # We first subset the intbc table to the cells in this clone,
648
+ # and to the intBCs in this clone (since we know they are high
649
+ # quality intBCs at this point, i.e. they do not contain sequencing
650
+ # errors nor homoplastic intBCs).
651
+ intbc_table_per_sample_for_clone = {
652
+ sample: intbc_table.drop(
653
+ intbc_table[
654
+ (~intbc_table["intBC"].isin(intbc_set_to_disconnect))
655
+ | (~intbc_table["cellBC"].isin(cells_in_clone_per_sample[sample]))
656
+ ].index
657
+ )
658
+ for sample, intbc_table in original_intbc_table_per_sample.items()
659
+ }
660
+ if _is_empty(intbc_table_per_sample_for_clone):
661
+ logger.info(
662
+ "intbc_table_per_sample_for_clone empty (case #1). Not recursing on clone."
663
+ )
664
+ recursive_clone_call[clone] = None
665
+ recursive_clone_call_integration_hierarchy[clone] = (
666
+ clone_intbc_set, # Note that we use the clone intBC set, to make sure we include everything.
667
+ None,
668
+ )
669
+ continue
670
+ # Filter intBCs based on min_intbc_umi_count
671
+ intbc_table_per_sample_for_clone = prune_low_umi_count_intbcs(
672
+ intbc_table_per_sample=intbc_table_per_sample_for_clone,
673
+ min_intbc_umi_count=min_intbc_umi_count,
674
+ ).intbc_table_per_sample
675
+ if _is_empty(intbc_table_per_sample_for_clone):
676
+ logger.info(
677
+ "intbc_table_per_sample_for_clone empty (case #2). Not recursing on clone."
678
+ )
679
+ recursive_clone_call[clone] = None
680
+ recursive_clone_call_integration_hierarchy[clone] = (
681
+ clone_intbc_set, # Note that we use the clone intBC set, to make sure we include everything.
682
+ None,
683
+ )
684
+ continue
685
+ # Filter intBCs based on `custom_intbc_filtering_function`
686
+ intbc_table_per_sample_for_clone = prune_custom_intbcs(
687
+ intbc_table_per_sample=intbc_table_per_sample_for_clone,
688
+ custom_intbc_filtering_function=custom_intbc_filtering_function,
689
+ ).intbc_table_per_sample
690
+ if _is_empty(intbc_table_per_sample_for_clone):
691
+ logger.info(
692
+ "intbc_table_per_sample_for_clone empty (case #3). Not recursing on clone."
693
+ )
694
+ recursive_clone_call[clone] = None
695
+ recursive_clone_call_integration_hierarchy[clone] = (
696
+ clone_intbc_set, # Note that we use the clone intBC set, to make sure we include everything.
697
+ None,
698
+ )
699
+ continue
700
+ # Filter intBCs based on `min_intbc_cellbc_count`
701
+ intbc_table_per_sample_for_clone = prune_low_cellbc_count_intbcs(
702
+ intbc_table_per_sample=intbc_table_per_sample_for_clone,
703
+ min_intbc_cellbc_count=min_intbc_cellbc_count,
704
+ ).intbc_table_per_sample
705
+ if _is_empty(intbc_table_per_sample_for_clone):
706
+ logger.info(
707
+ "intbc_table_per_sample_for_clone empty (case #4). Not recursing on clone."
708
+ )
709
+ recursive_clone_call[clone] = None
710
+ recursive_clone_call_integration_hierarchy[clone] = (
711
+ clone_intbc_set, # Note that we use the clone intBC set, to make sure we include everything.
712
+ None,
713
+ )
714
+ continue
715
+ # Build intBC graph
716
+ intbc_graph_for_clone = build_intbc_graph(
717
+ intbc_table_per_sample=intbc_table_per_sample_for_clone,
718
+ ).intbc_graph
719
+ # Keep reliable edges only
720
+ intbc_graph_for_clone = keep_reliable_edges(
721
+ intbc_graph=intbc_graph_for_clone,
722
+ ratio_cutoff=_reliable_edges_step__ratio_cutoff,
723
+ ).intbc_graph
724
+ progenitor_intbc_set = set()
725
+ all_intbcs_popped = False
726
+ while _is_connected(intbc_graph_for_clone):
727
+ if len(intbc_graph_for_clone[1]) == 0:
728
+ # We have popped all the intBCs. I.e. there was no subclonal structure left.
729
+ all_intbcs_popped = True
730
+ break
731
+ intbc, intbc_graph_for_clone = _pop_highest_frequency_vertex(
732
+ intbc_graph_for_clone
733
+ )
734
+ progenitor_intbc_set.add(intbc)
735
+ if all_intbcs_popped or len(progenitor_intbc_set) == 0:
736
+ # There was no subclonal structure left.
737
+ recursive_clone_call[clone] = None
738
+ recursive_clone_call_integration_hierarchy[clone] = (
739
+ clone_intbc_set, # Note that we use the clone intBC set, to make sure we include everything.
740
+ None,
741
+ )
742
+ logger.info(
743
+ "All intBCs popped when trying to identify progenitor intBC set, or "
744
+ "progenitor_intbc_set empty. No subclonal structure found. Will not "
745
+ "try to recurse."
746
+ )
747
+ continue
748
+ # If we are here, it means we were able to disconnect the graph.
749
+ # Therefore, unless the two (or more) components are aliased, we have a subclonal structure
750
+ # which we are ready to uncover using the recursive call.
751
+ logger.info(
752
+ f"Putative progenitor intBC set found (will proceed to call clones recursively excluding these intBCs): {progenitor_intbc_set}"
753
+ )
754
+
755
+ # Phase 2: Remove progenitor intBC set, and call recursively.
756
+ # Note that in the recursive call, we can just retain the intBCs
757
+ # in the clone's intBC set. This ensures there are no more
758
+ # sequencing errors nor homoplastic intBCs, cimplifying the
759
+ # recursive clone call.
760
+ # TODO: This might be very slow - use groupby instead to precompute it for all clones?
761
+ intbc_table_per_sample_for_clone = {
762
+ sample: intbc_table.drop(
763
+ intbc_table[
764
+ (intbc_table["intBC"].isin(progenitor_intbc_set))
765
+ | (~intbc_table["cellBC"].isin(cells_in_clone_per_sample[sample]))
766
+ ].index
767
+ )
768
+ for sample, intbc_table in original_intbc_table_per_sample.items()
769
+ }
770
+ recursive_clone_call__for_this_clone = clone_calling_algorithm(
771
+ intbc_table_per_sample=intbc_table_per_sample_for_clone, # This no longer contains sequencing errors not homoplastic intBCs.
772
+ min_intbc_umi_count=min_intbc_umi_count, # OK (still need it to discard intra-clone background RNA)
773
+ custom_intbc_filtering_function=custom_intbc_filtering_function, # Doesn't matter (OK)
774
+ min_intbc_cellbc_count=min_intbc_cellbc_count, # OK? Can consider removing this filter at this point?
775
+ edit_distance_cutoff=edit_distance_cutoff, # Doesn't actually matter since there are no sequencing errors at this point
776
+ _sequencing_error_pruning_step__co_occurrence_cutoff=_sequencing_error_pruning_step__co_occurrence_cutoff, # Doesn't actually matter since there are no sequencing errors at this point
777
+ _homoplastic_pruning_step__p_max_cutoff=-1, # Technically not needed since recover_single_intbc_clones will force it to -1 anyway.
778
+ _reliable_edges_step__ratio_cutoff=_reliable_edges_step__ratio_cutoff, # OK (still need it)
779
+ _preliminary_coreset_building_step__min_coreset_size=1, # Since there is no homoplasy eny more, can set it to 1.
780
+ _preliminary_coreset_building_step__min_overlap_for_merging_coresets=1, # Basically, we are just looking for connected components at this point.
781
+ _coreset_dealiasing_step__min_overlap_for_assigning_cellbc=1, # Again, there is no homoplasy, so we can set it to 1.
782
+ recover_single_intbc_clones=True, # Since we allow for single-intBC clones.
783
+ _coreset_dealiasing_step__co_occurrence_cutoff=_coreset_dealiasing_step__co_occurrence_cutoff, # Technically, there should be no aliasing since that's how we found the progenitor_intbc_set.
784
+ min_cells_assigned=recursive_min_cells_assigned, # OK - we allow for a smaller value to recover smaller subclones.
785
+ _coreset_disjointing_step__make_disjoint=_coreset_disjointing_step__make_disjoint, # Doesn't matter - there should be no homoplasy at this point.
786
+ _final_cell_assignment_step__min_overlap_for_assigning_cellbc=1, # Since there is no homoplasy eny more, can set it to 1.
787
+ _call_clones_from_coresets_step__min_intbc_counts_for_adding_intbc_to_clone_intbc_set=_call_clones_from_coresets_step__min_intbc_counts_for_adding_intbc_to_clone_intbc_set, # OK? Can consider lowering this.
788
+ min_intbc_fraction=min_intbc_fraction, # OK.
789
+ edit_distance_cutoff_phase_2=edit_distance_cutoff_phase_2, # Doesn't actually matter since there are no sequencing errors at this point
790
+ recursive=True,
791
+ recursive_min_fraction_uniquely_mapped_cells_to_recurse=recursive_min_fraction_uniquely_mapped_cells_to_recurse,
792
+ recursive_min_cells_assigned=recursive_min_cells_assigned,
793
+ )
794
+ # Check how many cells were mapped in the recursive clone call, to determine if we should retain the recursive split.
795
+ num_subclones_found = len(
796
+ recursive_clone_call__for_this_clone.clone_call.clone_to_cells_mapping
797
+ )
798
+ logger.info(
799
+ f"Found {num_subclones_found} clones in recursive call, with intBC sets: {recursive_clone_call__for_this_clone.clone_call.clone_intbc_sets}"
800
+ )
801
+ uniquely_mapped_cells_in_recursive_clone_call = sum(
802
+ [
803
+ len(cells_in_subclone)
804
+ for cells_in_subclone in recursive_clone_call__for_this_clone.clone_call.clone_to_cells_mapping.values()
805
+ ]
806
+ )
807
+ fraction_uniquely_mapped_cells_in_recursive_clone_call = (
808
+ uniquely_mapped_cells_in_recursive_clone_call / len(cells_in_clone)
809
+ )
810
+ logger.info(
811
+ f"fraction_uniquely_mapped_cells_in_recursive_clone_call = {fraction_uniquely_mapped_cells_in_recursive_clone_call}"
812
+ )
813
+ if (
814
+ len(
815
+ recursive_clone_call__for_this_clone.clone_call.clone_to_cells_mapping
816
+ )
817
+ == 1
818
+ ):
819
+ logger.info(
820
+ f"While is able to disconnect the coreset intBC graph, I was only able to find one subclone. This may be because either (1) the coresets were aliased, or (2) the subclonal structure was too small to be detected. Hence I will not recurse and just retain the original clone."
821
+ )
822
+ recursive_clone_call[clone] = None
823
+ recursive_clone_call_integration_hierarchy[clone] = (
824
+ clone_intbc_set, # Note that we use the clone intBC set, to make sure we include everything.
825
+ None,
826
+ )
827
+ elif (
828
+ len(
829
+ recursive_clone_call__for_this_clone.clone_call.clone_to_cells_mapping
830
+ )
831
+ >= 2
832
+ ):
833
+ if (
834
+ fraction_uniquely_mapped_cells_in_recursive_clone_call
835
+ >= recursive_min_fraction_uniquely_mapped_cells_to_recurse
836
+ ):
837
+ logger.info(
838
+ f"Recursive clone call found {num_subclones_found} subclones, and enough cells were uniquely mapped to these subclones ({fraction_uniquely_mapped_cells_in_recursive_clone_call}). Accepting the recursive decomposition."
839
+ )
840
+ recursive_clone_call[clone] = recursive_clone_call__for_this_clone
841
+ recursive_clone_call_integration_hierarchy[clone] = (
842
+ progenitor_intbc_set,
843
+ recursive_clone_call__for_this_clone.recursive_clone_call_integration_hierarchy,
844
+ )
845
+ else:
846
+ logger.info(
847
+ f"Recursive clone call found {num_subclones_found} subclones, but not enough cells were uniquely mapped to these subclones ({fraction_uniquely_mapped_cells_in_recursive_clone_call}). Hence I will not recurse and just retain the original clone."
848
+ )
849
+ recursive_clone_call[clone] = None
850
+ recursive_clone_call_integration_hierarchy[clone] = (
851
+ clone_intbc_set, # Note that we use the clone intBC set, to make sure we include everything.
852
+ None,
853
+ )
854
+ else:
855
+ # No clones found.
856
+ logger.info(
857
+ f"No subclonal structure found (i.e.). Hence I will not recurse and just retain the original clone."
858
+ )
859
+ recursive_clone_call[clone] = None
860
+ recursive_clone_call_integration_hierarchy[clone] = (
861
+ clone_intbc_set, # Note that we use the clone intBC set, to make sure we include everything.
862
+ None,
863
+ )
864
+ else:
865
+ recursive_clone_call = None
866
+ recursive_clone_call_integration_hierarchy = None
867
+
868
+ sanity_check_clone_call(clone_call)
869
+
870
+ total_time = time.time() - st
871
+ res = CloneCallingAlgorithmReturnType(
872
+ clone_call=clone_call,
873
+ recursive_clone_call=recursive_clone_call,
874
+ recursive_clone_call_integration_hierarchy=recursive_clone_call_integration_hierarchy,
875
+ total_time=total_time,
876
+ metadata={
877
+ "pruned_sequencing_errors": pruned_sequencing_errors,
878
+ },
879
+ )
880
+ return res
881
+
882
+
883
+ def _get_clone_to_cellbcs(
884
+ clone_call: CloneCallType,
885
+ ) -> Dict[CloneType, List[CellBCType]]:
886
+ """
887
+ Return the set of cellBCs mapping (uniquely, i.e. eliminating multiplets)
888
+ to each clone.
889
+ """
890
+ res = defaultdict(list)
891
+ for cellbc, clones in clone_call.cell_to_clones_mapping.items():
892
+ if len(clones) == 1: # I.e. we ignore multiplets
893
+ clone = list(clones)[0]
894
+ res[clone].append(cellbc)
895
+ return res
896
+
897
+
898
+ def _check_are_clones(
899
+ clone_call: CloneCallType, expected_clones: List[List[CellBCType]]
900
+ ) -> None:
901
+ """
902
+ Raise error if the clone_call does not have the .
903
+ """
904
+ clone_to_cellbcs = _get_clone_to_cellbcs(clone_call=clone_call)
905
+ expected_clones = sorted([sorted(x) for x in expected_clones])
906
+ actual_clones = sorted([sorted(x) for x in clone_to_cellbcs.values()])
907
+ if expected_clones != actual_clones:
908
+ raise ValueError(
909
+ f"Expected clones:\n"
910
+ f"{expected_clones}\n"
911
+ f"but actual clones:\n"
912
+ f"{actual_clones}"
913
+ )
914
+
915
+
916
+ def test_clone_calling_algorithm_mini():
917
+ """
918
+ Just tests that the clone calling algorithm runs on a trivial case.
919
+
920
+ In this test case, there are two clones characterized by the intBC sets
921
+ ["AAA", "BBB", "CCC"] and ["DDD", "EEE", "FFF"] respectively. The first
922
+ clone contains the cellBCs ["cellBC_1", "cellBC_2"] whereas the second
923
+ clone contains the cellBCs ["cellBC_3", "cellBC_4", "cellBC_5"].
924
+
925
+ Uniquely assigned cells should have ``cell_intbc_sets`` equal to the
926
+ intersection of their observed intBCs with their clone's intBC set.
927
+ """
928
+ intbc_table = pd.DataFrame(
929
+ data=[
930
+ ["cellBC_1", "AAA", 40],
931
+ ["cellBC_1", "BBB", 40],
932
+ # ["cellBC_1", "CCC", 40], # Exclude this to make test more interesting.
933
+ ["cellBC_2", "AAA", 40],
934
+ ["cellBC_2", "BBB", 40],
935
+ ["cellBC_2", "CCC", 40],
936
+ ["cellBC_3", "DDD", 40],
937
+ ["cellBC_3", "EEE", 40],
938
+ ["cellBC_3", "FFF", 40],
939
+ ["cellBC_4", "DDD", 40],
940
+ ["cellBC_4", "EEE", 40],
941
+ ["cellBC_4", "FFF", 40],
942
+ ["cellBC_5", "DDD", 9], # Low UMI count to make the test more interesting.
943
+ ["cellBC_5", "EEE", 40],
944
+ ["cellBC_5", "FFF", 40],
945
+ ["cellBC_5", "FFH", 15], # Sequencing error.
946
+ ],
947
+ columns=["cellBC", "intBC", "UMI_count"],
948
+ )
949
+
950
+ clone_call = clone_calling_algorithm(
951
+ intbc_table_per_sample={"sample1": intbc_table},
952
+ min_intbc_cellbc_count=1,
953
+ edit_distance_cutoff=1,
954
+ min_cells_assigned=1,
955
+ # Collapse 1-edit neighbors of an already-kept intBC (drops FFH vs FFF).
956
+ edit_distance_cutoff_phase_2=1,
957
+ ).clone_call
958
+ _check_are_clones(
959
+ clone_call=clone_call,
960
+ expected_clones=[
961
+ {"sample1_cellBC_1", "sample1_cellBC_2"},
962
+ {"sample1_cellBC_3", "sample1_cellBC_4", "sample1_cellBC_5"},
963
+ ],
964
+ )
965
+ assert clone_call.clone_intbc_sets["clone_0"] == set(["AAA", "BBB", "CCC"])
966
+ assert clone_call.clone_intbc_sets["clone_1"] == set(["DDD", "EEE", "FFF"])
967
+ assert clone_call.cell_to_clones_mapping["sample1_cellBC_1"] == set(["clone_0"])
968
+ assert clone_call.cell_to_clones_mapping["sample1_cellBC_2"] == set(["clone_0"])
969
+ assert clone_call.cell_to_clones_mapping["sample1_cellBC_3"] == set(["clone_1"])
970
+ assert clone_call.cell_to_clones_mapping["sample1_cellBC_4"] == set(["clone_1"])
971
+ assert clone_call.cell_to_clones_mapping["sample1_cellBC_5"] == set(["clone_1"])
972
+
973
+ clone0_intbcs = {"AAA", "BBB", "CCC"}
974
+ clone1_intbcs = {"DDD", "EEE", "FFF"}
975
+ assert clone_call.cell_intbc_sets["sample1_cellBC_1"] == {"AAA", "BBB"}
976
+ assert clone_call.cell_intbc_sets["sample1_cellBC_2"] == clone0_intbcs
977
+ assert clone_call.cell_intbc_sets["sample1_cellBC_3"] == clone1_intbcs
978
+ assert clone_call.cell_intbc_sets["sample1_cellBC_4"] == clone1_intbcs
979
+ assert clone_call.cell_intbc_sets["sample1_cellBC_5"] == {"EEE", "FFF"}
980
+
981
+
982
+ def test_recursive_clone_calling_algorithm_mini():
983
+ """
984
+ Just tests that the resursive clone calling algorithm runs on a small case.
985
+
986
+ Subclone structure is:
987
+
988
+ root
989
+ |
990
+ A,B (first round of integration)
991
+ |
992
+ / \
993
+ C D (second round of integration)
994
+ | |
995
+ ABC ABD
996
+ """
997
+ logger = logging.getLogger(__name__)
998
+ logger.setLevel(0)
999
+
1000
+ intbc_table = pd.DataFrame(
1001
+ data=[
1002
+ ["cellBC_1", "AAA", 40],
1003
+ ["cellBC_1", "BBB", 40],
1004
+ ["cellBC_1", "CCC", 40],
1005
+ ["cellBC_11", "AAA", 40],
1006
+ ["cellBC_11", "BBB", 40],
1007
+ ["cellBC_11", "CCC", 40],
1008
+ ["cellBC_2", "AAA", 40],
1009
+ ["cellBC_2", "BBB", 40],
1010
+ ["cellBC_2", "CCC", 40],
1011
+ ["cellBC_22", "AAA", 40],
1012
+ ["cellBC_22", "BBB", 40],
1013
+ ["cellBC_22", "CCC", 40],
1014
+ ["cellBC_3", "AAA", 40],
1015
+ ["cellBC_3", "BBB", 40],
1016
+ ["cellBC_3", "DDD", 40],
1017
+ ["cellBC_33", "AAA", 40],
1018
+ ["cellBC_33", "BBB", 40],
1019
+ ["cellBC_33", "DDD", 40],
1020
+ ["cellBC_4", "AAA", 40],
1021
+ ["cellBC_4", "BBB", 40],
1022
+ ["cellBC_4", "DDD", 40],
1023
+ ["cellBC_44", "AAA", 40],
1024
+ ["cellBC_44", "BBB", 40],
1025
+ ["cellBC_44", "DDD", 40],
1026
+ ["cellBC_5", "AAA", 40], # Cell must be unmapped in the recursion
1027
+ ["cellBC_5", "BBB", 40],
1028
+ ["cellBC_6", "AAA", 40], # Intra-clonal doublet.
1029
+ ["cellBC_6", "BBB", 40],
1030
+ ["cellBC_6", "CCC", 40],
1031
+ ["cellBC_6", "DDD", 40],
1032
+ ],
1033
+ columns=["cellBC", "intBC", "UMI_count"],
1034
+ )
1035
+
1036
+ clone_call_namedtuple = clone_calling_algorithm(
1037
+ intbc_table_per_sample={"sample1": intbc_table},
1038
+ min_intbc_cellbc_count=1,
1039
+ edit_distance_cutoff=0,
1040
+ min_cells_assigned=1,
1041
+ recursive=True,
1042
+ recursive_min_cells_assigned=2,
1043
+ )
1044
+ clone_call = clone_call_namedtuple.clone_call
1045
+ recursive_clone_call = clone_call_namedtuple.recursive_clone_call
1046
+ _check_are_clones(
1047
+ clone_call=clone_call,
1048
+ expected_clones=[
1049
+ [
1050
+ "sample1_cellBC_1",
1051
+ "sample1_cellBC_11",
1052
+ "sample1_cellBC_2",
1053
+ "sample1_cellBC_22",
1054
+ "sample1_cellBC_3",
1055
+ "sample1_cellBC_33",
1056
+ "sample1_cellBC_4",
1057
+ "sample1_cellBC_44",
1058
+ "sample1_cellBC_5",
1059
+ "sample1_cellBC_6",
1060
+ ],
1061
+ ],
1062
+ )
1063
+ _check_are_clones(
1064
+ clone_call=recursive_clone_call["clone_0"].clone_call,
1065
+ expected_clones=[
1066
+ [
1067
+ "sample1_cellBC_1",
1068
+ "sample1_cellBC_11",
1069
+ "sample1_cellBC_2",
1070
+ "sample1_cellBC_22",
1071
+ ],
1072
+ [
1073
+ "sample1_cellBC_3",
1074
+ "sample1_cellBC_33",
1075
+ "sample1_cellBC_4",
1076
+ "sample1_cellBC_44",
1077
+ ],
1078
+ ],
1079
+ )
1080
+ print(
1081
+ f"clone_call_namedtuple.recursive_clone_call_integration_hierarchy = {clone_call_namedtuple.recursive_clone_call_integration_hierarchy}"
1082
+ )
1083
+ assert clone_call_namedtuple.recursive_clone_call_integration_hierarchy == {
1084
+ "clone_0": (
1085
+ {"AAA", "BBB"},
1086
+ {
1087
+ "clone_0": ({"CCC"}, None),
1088
+ "clone_1": ({"DDD"}, None),
1089
+ },
1090
+ ),
1091
+ }
1092
+
1093
+
1094
+ def test_recursive_clone_calling_algorithm_larger():
1095
+ """
1096
+ Same as test_recursive_clone_calling_algorithm_mini but with many cells.
1097
+
1098
+ Subclone structure is:
1099
+
1100
+ root
1101
+ |
1102
+ A,B (first round of integration)
1103
+ |
1104
+ / \
1105
+ C D (second round of integration)
1106
+ | |
1107
+ ABC ABD
1108
+ """
1109
+ logger = logging.getLogger(__name__)
1110
+ logger.setLevel(0)
1111
+
1112
+ n_cell_copies = 10
1113
+
1114
+ intbc_table = pd.DataFrame(
1115
+ data=[
1116
+ [f"cellBC_1_{i}", "AAA", 40] for i in range(n_cell_copies)
1117
+ ] + [
1118
+ [f"cellBC_1_{i}", "BBB", 40] for i in range(n_cell_copies)
1119
+ ] + [
1120
+ [f"cellBC_1_{i}", "CCC", 40] for i in range(n_cell_copies)
1121
+ ] + [
1122
+ [f"cellBC_11_{i}", "AAA", 40] for i in range(n_cell_copies)
1123
+ ] + [
1124
+ [f"cellBC_11_{i}", "BBB", 40] for i in range(n_cell_copies)
1125
+ ] + [
1126
+ [f"cellBC_11_{i}", "CCC", 40] for i in range(n_cell_copies)
1127
+ ] + [
1128
+ [f"cellBC_2_{i}", "AAA", 40] for i in range(n_cell_copies)
1129
+ ] + [
1130
+ [f"cellBC_2_{i}", "BBB", 40] for i in range(n_cell_copies)
1131
+ ] + [
1132
+ [f"cellBC_2_{i}", "CCC", 40] for i in range(n_cell_copies)
1133
+ ] + [
1134
+ [f"cellBC_22_{i}", "AAA", 40] for i in range(n_cell_copies)
1135
+ ] + [
1136
+ [f"cellBC_22_{i}", "BBB", 40] for i in range(n_cell_copies)
1137
+ ] + [
1138
+ [f"cellBC_22_{i}", "CCC", 40] for i in range(n_cell_copies)
1139
+ ] + [
1140
+ [f"cellBC_3_{i}", "AAA", 40] for i in range(n_cell_copies)
1141
+ ] + [
1142
+ [f"cellBC_3_{i}", "BBB", 40] for i in range(n_cell_copies)
1143
+ ] + [
1144
+ [f"cellBC_3_{i}", "DDD", 40] for i in range(n_cell_copies)
1145
+ ] + [
1146
+ [f"cellBC_33_{i}", "AAA", 40] for i in range(n_cell_copies)
1147
+ ] + [
1148
+ [f"cellBC_33_{i}", "BBB", 40] for i in range(n_cell_copies)
1149
+ ] + [
1150
+ [f"cellBC_33_{i}", "DDD", 40] for i in range(n_cell_copies)
1151
+ ] + [
1152
+ [f"cellBC_4_{i}", "AAA", 40] for i in range(n_cell_copies)
1153
+ ] + [
1154
+ [f"cellBC_4_{i}", "BBB", 40] for i in range(n_cell_copies)
1155
+ ] + [
1156
+ [f"cellBC_4_{i}", "DDD", 40] for i in range(n_cell_copies)
1157
+ ] + [
1158
+ [f"cellBC_44_{i}", "AAA", 40] for i in range(n_cell_copies)
1159
+ ] + [
1160
+ [f"cellBC_44_{i}", "BBB", 40] for i in range(n_cell_copies)
1161
+ ] + [
1162
+ [f"cellBC_44_{i}", "DDD", 40] for i in range(n_cell_copies)
1163
+ ] + [
1164
+ [f"cellBC_5_{i}", "AAA", 40] for i in range(n_cell_copies)
1165
+ ] + [
1166
+ [f"cellBC_5_{i}", "BBB", 40] for i in range(n_cell_copies)
1167
+ ] + [
1168
+ [f"cellBC_6_{i}", "AAA", 40] for i in range(n_cell_copies)
1169
+ ] + [
1170
+ [f"cellBC_6_{i}", "BBB", 40] for i in range(n_cell_copies)
1171
+ ] + [
1172
+ [f"cellBC_6_{i}", "CCC", 40] for i in range(n_cell_copies)
1173
+ ] + [
1174
+ [f"cellBC_6_{i}", "DDD", 40] for i in range(n_cell_copies)
1175
+ ],
1176
+ columns=["cellBC", "intBC", "UMI_count"],
1177
+ )
1178
+
1179
+ clone_call_namedtuple = clone_calling_algorithm(
1180
+ intbc_table_per_sample={"sample1": intbc_table},
1181
+ edit_distance_cutoff=0,
1182
+ recursive=True,
1183
+ )
1184
+ clone_call = clone_call_namedtuple.clone_call
1185
+ recursive_clone_call = clone_call_namedtuple.recursive_clone_call
1186
+ _check_are_clones(
1187
+ clone_call=clone_call,
1188
+ expected_clones=[ # all cells
1189
+ [f"sample1_cellBC_1_{i}" for i in range(n_cell_copies)]
1190
+ + [f"sample1_cellBC_11_{i}" for i in range(n_cell_copies)]
1191
+ + [f"sample1_cellBC_2_{i}" for i in range(n_cell_copies)]
1192
+ + [f"sample1_cellBC_22_{i}" for i in range(n_cell_copies)]
1193
+ + [f"sample1_cellBC_3_{i}" for i in range(n_cell_copies)]
1194
+ + [f"sample1_cellBC_33_{i}" for i in range(n_cell_copies)]
1195
+ + [f"sample1_cellBC_4_{i}" for i in range(n_cell_copies)]
1196
+ + [f"sample1_cellBC_44_{i}" for i in range(n_cell_copies)]
1197
+ + [f"sample1_cellBC_5_{i}" for i in range(n_cell_copies)]
1198
+ + [f"sample1_cellBC_6_{i}" for i in range(n_cell_copies)]
1199
+ ],
1200
+ )
1201
+ _check_are_clones(
1202
+ clone_call=recursive_clone_call["clone_0"].clone_call,
1203
+ expected_clones=[[]
1204
+ + [f"sample1_cellBC_1_{i}" for i in range(n_cell_copies)]
1205
+ + [f"sample1_cellBC_11_{i}" for i in range(n_cell_copies)]
1206
+ + [f"sample1_cellBC_2_{i}" for i in range(n_cell_copies)]
1207
+ + [f"sample1_cellBC_22_{i}" for i in range(n_cell_copies)]
1208
+ ] + [ []
1209
+ + [f"sample1_cellBC_3_{i}" for i in range(n_cell_copies)]
1210
+ + [f"sample1_cellBC_33_{i}" for i in range(n_cell_copies)]
1211
+ + [f"sample1_cellBC_4_{i}" for i in range(n_cell_copies)]
1212
+ + [f"sample1_cellBC_44_{i}" for i in range(n_cell_copies)]
1213
+ ],
1214
+ )
1215
+ print(
1216
+ f"clone_call_namedtuple.recursive_clone_call_integration_hierarchy = {clone_call_namedtuple.recursive_clone_call_integration_hierarchy}"
1217
+ )
1218
+ assert clone_call_namedtuple.recursive_clone_call_integration_hierarchy == {
1219
+ "clone_0": (
1220
+ {"AAA", "BBB"},
1221
+ {
1222
+ "clone_0": ({"CCC"}, None),
1223
+ "clone_1": ({"DDD"}, None),
1224
+ },
1225
+ ),
1226
+ }
1227
+
1228
+
1229
+ def test_recursive_clone_calling_algorithm_on_real_data():
1230
+ """
1231
+ Tests the recursive clone calling algorithm on real data.
1232
+ """
1233
+ res = clone_calling_algorithm(
1234
+ intbc_table_per_sample={
1235
+ "3B": pd.read_csv(
1236
+ "notebooks/test_data/3B_clone_11_intBC_table_per_sample.csv"
1237
+ )
1238
+ },
1239
+ recursive=True,
1240
+ )
1241
+
1242
+ clone_to_cells_mapping = res.clone_call.clone_to_cells_mapping
1243
+ clone_intbc_sets = res.clone_call.clone_intbc_sets
1244
+ cell_intbc_sets = res.clone_call.cell_intbc_sets
1245
+
1246
+ clone = "clone_0"
1247
+
1248
+ cells_in_clone = clone_to_cells_mapping[clone]
1249
+
1250
+ intbc_co_counts = pd.DataFrame(
1251
+ np.zeros(shape=(len(clone_intbc_sets[clone]), len(clone_intbc_sets[clone]))),
1252
+ index=sorted(list(clone_intbc_sets[clone])),
1253
+ columns=sorted(list(clone_intbc_sets[clone])),
1254
+ )
1255
+
1256
+ for cell in cells_in_clone:
1257
+ cell_intbc_list = list(cell_intbc_sets[cell])
1258
+ for i in range(len(cell_intbc_list)):
1259
+ intbc_i = cell_intbc_list[i]
1260
+ for j in range(len(cell_intbc_list)):
1261
+ intbc_j = cell_intbc_list[j]
1262
+ intbc_co_counts.loc[intbc_i, intbc_j] += 0.5
1263
+ intbc_co_counts.loc[intbc_j, intbc_i] += 0.5
1264
+
1265
+ g = sns.clustermap(intbc_co_counts)
1266
+
1267
+ ### This would show the co-occurrence matrix with the rows and columns reordered
1268
+ ### according to the hierarchical clustering.
1269
+ # # Get the reordered row and column indices
1270
+ # row_order = g.dendrogram_row.reordered_ind
1271
+ # col_order = g.dendrogram_col.reordered_ind
1272
+
1273
+ # # Reorder the matrix
1274
+ # reordered_matrix = intbc_co_counts.iloc[row_order, col_order]
1275
+
1276
+ # # Print or use the reordered matrix
1277
+ # print(reordered_matrix)
1278
+
1279
+ assert res.recursive_clone_call_integration_hierarchy == {
1280
+ "clone_0": (
1281
+ {
1282
+ "CCTATGGAAAATGT",
1283
+ "CGTCGTAATCGTCT",
1284
+ "GCCCCTAATATAAG",
1285
+ "GCTAGCGACGTTCG",
1286
+ "TTTGCCCTGCAAAC",
1287
+ },
1288
+ {
1289
+ "clone_0": (
1290
+ {
1291
+ "AAATGAGCATTACG",
1292
+ "CGGTTCAACGAGTA",
1293
+ "GTAGTGACTGACTG",
1294
+ "TCTTCAGTTTTAAT",
1295
+ },
1296
+ None,
1297
+ ),
1298
+ "clone_1": (
1299
+ {
1300
+ "ACATTGAACATCTT",
1301
+ "ACATTGGAGGCCGA",
1302
+ "ACTATTAATCTGCT",
1303
+ "GGGCCCCGCGCTTA",
1304
+ },
1305
+ None,
1306
+ ),
1307
+ "clone_2": (
1308
+ {"ATTCCTGCTGTACG", "CTTCAACCATTTTG", "GTTGAGTGTCGTCT"},
1309
+ None,
1310
+ ),
1311
+ "clone_3": (
1312
+ {
1313
+ "CAAAATCTGATAAA",
1314
+ "CTATTTTCATTAAT",
1315
+ "GCTACCTGCCAGCG",
1316
+ "GTCCCCCAATCTGC",
1317
+ "TTCATTTCTTACGA",
1318
+ },
1319
+ None,
1320
+ ),
1321
+ },
1322
+ )
1323
+ }
1324
+
1325
+
1326
+ def test_clone_calling_algorithm_doublet():
1327
+ ncells_per_clone = 100
1328
+ intbc_table = pd.DataFrame(
1329
+ data=[[f"cellBC_{i}", "AAA", 40] for i in range(ncells_per_clone)]
1330
+ + [[f"cellBC_{i}", "BBB", 40] for i in range(ncells_per_clone)]
1331
+ + [[f"cellBC_{i}", "CCC", 40] for i in range(ncells_per_clone)]
1332
+ + [
1333
+ [f"cellBC_{i}", "DDD", 40]
1334
+ for i in range(ncells_per_clone, 2 * ncells_per_clone)
1335
+ ]
1336
+ + [
1337
+ [f"cellBC_{i}", "EEE", 40]
1338
+ for i in range(ncells_per_clone, 2 * ncells_per_clone)
1339
+ ]
1340
+ + [
1341
+ [f"cellBC_{i}", "FFF", 40]
1342
+ for i in range(ncells_per_clone, 2 * ncells_per_clone)
1343
+ ]
1344
+ + [
1345
+ ["cellBC_doublet", "AAA", 40],
1346
+ ["cellBC_doublet", "BBB", 40],
1347
+ ["cellBC_doublet", "CCC", 40],
1348
+ ["cellBC_doublet", "DDD", 40],
1349
+ ["cellBC_doublet", "EEE", 40],
1350
+ ["cellBC_doublet", "FFF", 40],
1351
+ ],
1352
+ columns=["cellBC", "intBC", "UMI_count"],
1353
+ )
1354
+
1355
+ clone_call = clone_calling_algorithm(
1356
+ intbc_table_per_sample={"sample1": intbc_table},
1357
+ edit_distance_cutoff=0, # Turn off sequencing error correction
1358
+ ).clone_call
1359
+
1360
+ _check_are_clones(
1361
+ clone_call=clone_call,
1362
+ expected_clones=[
1363
+ [f"sample1_cellBC_{i}" for i in range(ncells_per_clone)],
1364
+ [
1365
+ f"sample1_cellBC_{i}"
1366
+ for i in range(ncells_per_clone, 2 * ncells_per_clone)
1367
+ ],
1368
+ ],
1369
+ )
1370
+ assert clone_call.cell_to_clones_mapping["sample1_cellBC_doublet"] == set(
1371
+ ["clone_0", "clone_1"]
1372
+ )
1373
+
1374
+
1375
+ def test_clone_calling_algorithm_single_intBC_clones():
1376
+ """
1377
+ NovaClone can recover single intBC clones too, by running with
1378
+ `recover_single_intbc_clones=True`.
1379
+ """
1380
+ ncells_per_clone = 100
1381
+ intbc_table = pd.DataFrame(
1382
+ data=[[f"cellBC_{i}", "AAA", 40] for i in range(ncells_per_clone)]
1383
+ + [[f"cellBC_{i}", "BBB", 40] for i in range(ncells_per_clone)]
1384
+ + [[f"cellBC_{i}", "CCC", 40] for i in range(ncells_per_clone)]
1385
+ + [
1386
+ [f"cellBC_{i}", "DDD", 40]
1387
+ for i in range(ncells_per_clone, 2 * ncells_per_clone)
1388
+ ]
1389
+ + [
1390
+ [f"cellBC_{i}", "EEE", 40]
1391
+ for i in range(ncells_per_clone, 2 * ncells_per_clone)
1392
+ ]
1393
+ + [
1394
+ [f"cellBC_{i}", "FFF", 40]
1395
+ for i in range(ncells_per_clone, 2 * ncells_per_clone)
1396
+ ] + [
1397
+ [f"cellBC_{i}", "GGG", 40]
1398
+ for i in range(2 * ncells_per_clone, 3 * ncells_per_clone)
1399
+ ] + [
1400
+ [f"cellBC_{i}", "HHH", 40]
1401
+ for i in range(3 * ncells_per_clone, 4 * ncells_per_clone)
1402
+ ]
1403
+ + [
1404
+ ["cellBC_doublet", "AAA", 40],
1405
+ ["cellBC_doublet", "BBB", 40],
1406
+ ["cellBC_doublet", "CCC", 40],
1407
+ ["cellBC_doublet", "DDD", 40],
1408
+ ["cellBC_doublet", "EEE", 40],
1409
+ ["cellBC_doublet", "FFF", 40],
1410
+ ] + [
1411
+ ["cellBC_doublet2", "AAA", 40],
1412
+ ["cellBC_doublet2", "BBB", 40],
1413
+ ["cellBC_doublet2", "CCC", 40],
1414
+ ["cellBC_doublet2", "GGG", 40],
1415
+ ] + [
1416
+ ["cellBC_doublet3", "GGG", 40],
1417
+ ["cellBC_doublet3", "HHH", 40],
1418
+ ] + [
1419
+ ["cellBC_doublet4", "AAA", 40],
1420
+ ["cellBC_doublet4", "BBB", 40],
1421
+ ["cellBC_doublet4", "GGG", 40],
1422
+ ["cellBC_doublet4", "HHH", 40],
1423
+ ] + [
1424
+ ["empty_droplet", "AAA", 1],
1425
+ ["empty_droplet", "BBB", 1],
1426
+ ["empty_droplet", "GGG", 1],
1427
+ ],
1428
+ columns=["cellBC", "intBC", "UMI_count"],
1429
+ )
1430
+
1431
+ clone_call = clone_calling_algorithm(
1432
+ intbc_table_per_sample={"sample1": intbc_table},
1433
+ edit_distance_cutoff=0, # Turn off sequencing error correction
1434
+ recover_single_intbc_clones=True,
1435
+ ).clone_call
1436
+
1437
+ _check_are_clones(
1438
+ clone_call=clone_call,
1439
+ expected_clones=[
1440
+ [f"sample1_cellBC_{i}" for i in range(ncells_per_clone)],
1441
+ [
1442
+ f"sample1_cellBC_{i}"
1443
+ for i in range(ncells_per_clone, 2 * ncells_per_clone)
1444
+ ],
1445
+ [
1446
+ f"sample1_cellBC_{i}"
1447
+ for i in range(2 * ncells_per_clone, 3 * ncells_per_clone)
1448
+ ],
1449
+ [
1450
+ f"sample1_cellBC_{i}"
1451
+ for i in range(3 * ncells_per_clone, 4 * ncells_per_clone)
1452
+ ],
1453
+ ],
1454
+ )
1455
+ assert clone_call.cell_to_clones_mapping["sample1_cellBC_doublet"] == set(
1456
+ ["clone_0", "clone_1"]
1457
+ )
1458
+ assert clone_call.cell_to_clones_mapping["sample1_cellBC_doublet2"] == set(
1459
+ ["clone_0", "clone_2"]
1460
+ )
1461
+ assert clone_call.cell_to_clones_mapping["sample1_cellBC_doublet3"] == set(
1462
+ ["clone_2", "clone_3"]
1463
+ )
1464
+ assert clone_call.cell_to_clones_mapping["sample1_cellBC_doublet4"] == set(
1465
+ ["clone_0", "clone_2", "clone_3"]
1466
+ )
1467
+ assert clone_call.cell_to_clones_mapping["sample1_empty_droplet"] == set(
1468
+ []
1469
+ )
1470
+
1471
+
1472
+ def test_clone_calling_algorithm_single_intBC_clones_trivial():
1473
+ """
1474
+ NovaClone can recover single intBC clones too, by running with
1475
+ `recover_single_intbc_clones=True`. Just one clone with one intBC.
1476
+ """
1477
+ ncells_per_clone = 100
1478
+ intbc_table = pd.DataFrame(
1479
+ data=[[f"cellBC_{i}", "AAA", 40] for i in range(ncells_per_clone)],
1480
+ columns=["cellBC", "intBC", "UMI_count"],
1481
+ )
1482
+
1483
+ clone_call = clone_calling_algorithm(
1484
+ intbc_table_per_sample={"sample1": intbc_table},
1485
+ edit_distance_cutoff=0, # Turn off sequencing error correction
1486
+ recover_single_intbc_clones=True,
1487
+ ).clone_call
1488
+
1489
+ _check_are_clones(
1490
+ clone_call=clone_call,
1491
+ expected_clones=[
1492
+ [f"sample1_cellBC_{i}" for i in range(ncells_per_clone)],
1493
+ ],
1494
+ )
1495
+
1496
+
1497
+ def test_clone_calling_algorithm_single_intBC_clones__border_case():
1498
+ f"""
1499
+ The pruned intBC graph has:
1500
+ ["a", "b"]
1501
+ ["a", "c"]
1502
+ ["d"]
1503
+ After making the coresets disjoint, this leads to coresets ["b"], ["c"], ["d"];
1504
+ - With a _final_cell_assignment_step__min_overlap_for_assigning_cellbc=1,
1505
+ the three clones remain, all doublets correctly identified.
1506
+ - With a _final_cell_assignment_step__min_overlap_for_assigning_cellbc=2,
1507
+ this should cause the ["b"] and ["c"] coresets to get no cells mapped (and thus drop off),
1508
+ versus ["d"] does get them since it's a single-intBC clone. However, ["d"] gets `cellBC_doublet2`
1509
+ mapped since it only overlaps in 1 intBC with the pruned coresets ["b"], ["c"].
1510
+ """
1511
+ ncells_per_clone = 100
1512
+ intbc_table = pd.DataFrame(
1513
+ data=[[f"cellBC_{i}", "AAA", 40] for i in range(ncells_per_clone)]
1514
+ + [[f"cellBC_{i}", "BBB", 40] for i in range(ncells_per_clone)]
1515
+ + [
1516
+ [f"cellBC_{i}", "AAA", 40]
1517
+ for i in range(ncells_per_clone, 2 * ncells_per_clone)
1518
+ ]
1519
+ + [
1520
+ [f"cellBC_{i}", "CCC", 40]
1521
+ for i in range(ncells_per_clone, 2 * ncells_per_clone)
1522
+ ]
1523
+ + [
1524
+ [f"cellBC_{i}", "DDD", 40]
1525
+ for i in range(2 * ncells_per_clone, 3 * ncells_per_clone)
1526
+ ]
1527
+ + [
1528
+ ["cellBC_doublet", "AAA", 40],
1529
+ ["cellBC_doublet", "BBB", 40],
1530
+ ["cellBC_doublet", "CCC", 40],
1531
+ ] + [
1532
+ ["cellBC_doublet2", "AAA", 40],
1533
+ ["cellBC_doublet2", "BBB", 40],
1534
+ ["cellBC_doublet2", "CCC", 40],
1535
+ ["cellBC_doublet2", "DDD", 40],
1536
+ ] + [
1537
+ ["empty_droplet", "AAA", 1],
1538
+ ["empty_droplet", "BBB", 1],
1539
+ ["empty_droplet", "GGG", 1],
1540
+ ],
1541
+ columns=["cellBC", "intBC", "UMI_count"],
1542
+ )
1543
+
1544
+ clone_call_1 = clone_calling_algorithm(
1545
+ intbc_table_per_sample={"sample1": intbc_table},
1546
+ edit_distance_cutoff=0, # Turn off sequencing error correction
1547
+ recover_single_intbc_clones=True,
1548
+ ).clone_call
1549
+
1550
+ _check_are_clones(
1551
+ clone_call=clone_call_1,
1552
+ expected_clones=[
1553
+ [f"sample1_cellBC_{i}" for i in range(ncells_per_clone)],
1554
+ [f"sample1_cellBC_{i}" for i in range(ncells_per_clone, 2 * ncells_per_clone)],
1555
+ [f"sample1_cellBC_{i}" for i in range(2 * ncells_per_clone, 3 * ncells_per_clone)],
1556
+ ],
1557
+ )
1558
+
1559
+ clone_call_2 = clone_calling_algorithm(
1560
+ intbc_table_per_sample={"sample1": intbc_table},
1561
+ edit_distance_cutoff=0, # Turn off sequencing error correction
1562
+ recover_single_intbc_clones=True,
1563
+ _final_cell_assignment_step__min_overlap_for_assigning_cellbc=2,
1564
+ ).clone_call
1565
+
1566
+ _check_are_clones(
1567
+ clone_call=clone_call_2,
1568
+ expected_clones=[
1569
+ [f"sample1_cellBC_{i}" for i in range(2 * ncells_per_clone, 3 * ncells_per_clone)] + ["sample1_cellBC_doublet2"],
1570
+ ],
1571
+ )