nova-clone 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. nova_clone/__init__.py +1 -0
  2. nova_clone/algorithm/__init__.py +2 -0
  3. nova_clone/algorithm/_build_intbc_graph.py +135 -0
  4. nova_clone/algorithm/_call_clones_from_coresets.py +235 -0
  5. nova_clone/algorithm/_clone_call_ops.py +258 -0
  6. nova_clone/algorithm/_clone_calling_algorithm.py +1571 -0
  7. nova_clone/algorithm/_compute_cellbc_vs_coreset_overlap_matrix.py +143 -0
  8. nova_clone/algorithm/_compute_coreset_co_assignment_matrix.py +170 -0
  9. nova_clone/algorithm/_find_maximal_coresets.py +428 -0
  10. nova_clone/algorithm/_keep_reliable_edges.py +202 -0
  11. nova_clone/algorithm/_make_coresets_disjoint.py +114 -0
  12. nova_clone/algorithm/_prune_coresets.py +164 -0
  13. nova_clone/algorithm/_prune_custom_intbcs.py +150 -0
  14. nova_clone/algorithm/_prune_likely_homoplastic_intbcs.py +162 -0
  15. nova_clone/algorithm/_prune_likely_sequencing_errors.py +201 -0
  16. nova_clone/algorithm/_prune_low_cellbc_count_intbcs.py +154 -0
  17. nova_clone/algorithm/_prune_low_umi_count_intbcs.py +152 -0
  18. nova_clone/algorithm/_types.py +150 -0
  19. nova_clone/algorithm/_utils.py +53 -0
  20. nova_clone/algorithm/cassiopeia/__init__.py +1 -0
  21. nova_clone/algorithm/cassiopeia/_call_lineage_groups.py +270 -0
  22. nova_clone/caching/__init__.py +24 -0
  23. nova_clone/caching/_cached.py +175 -0
  24. nova_clone/caching/_cached_computation.py +369 -0
  25. nova_clone/caching/_cached_parallel_computation.py +428 -0
  26. nova_clone/caching/_common.py +200 -0
  27. nova_clone/data/__init__.py +1 -0
  28. nova_clone/qc/__init__.py +179 -0
  29. nova_clone/qc/_clone_intbc_sets.py +234 -0
  30. nova_clone/qc/_compare_clone_calls.py +218 -0
  31. nova_clone/qc/_data_retention.py +175 -0
  32. nova_clone/qc/_homoplasy.py +81 -0
  33. nova_clone/qc/_inter_clonal_kinship.py +60 -0
  34. nova_clone/qc/_intra_clonal_kinship.py +172 -0
  35. nova_clone/qc/_multiplet_rate.py +158 -0
  36. nova_clone/qc/_number_of_clones.py +153 -0
  37. nova_clone/qc/_pre_qc.py +485 -0
  38. nova_clone/simulation/HDBSCAN_method_adjusted.py +68 -0
  39. nova_clone/simulation/__init__.py +0 -0
  40. nova_clone/simulation/_caching.py +40 -0
  41. nova_clone/simulation/benchmarking.py +1221 -0
  42. nova_clone/simulation/clone_to_sequences.py +408 -0
  43. nova_clone/simulation/creating_tree.py +416 -0
  44. nova_clone/simulation/evaluation.py +344 -0
  45. nova_clone/simulation/jaccard_hierarchical_by_intBC.py +78 -0
  46. nova_clone/simulation/plot_benchmark_lines.py +226 -0
  47. nova_clone/simulation/slice_benchmark.py +640 -0
  48. nova_clone/simulation/slice_benchmark_clone_calling.py +68 -0
  49. nova_clone-0.1.0.dist-info/METADATA +298 -0
  50. nova_clone-0.1.0.dist-info/RECORD +53 -0
  51. nova_clone-0.1.0.dist-info/WHEEL +5 -0
  52. nova_clone-0.1.0.dist-info/licenses/LICENSE +28 -0
  53. nova_clone-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,369 @@
1
+ import logging
2
+ import os
3
+ from copy import deepcopy
4
+ from functools import wraps
5
+ from inspect import signature
6
+ from typing import List
7
+
8
+ from ._common import (CacheUsageError, _get_func_caching_dir, _get_mode,
9
+ _validate_decorator_args, get_cache_dir, get_read_only,
10
+ get_use_hash)
11
+
12
+ logger = logging.getLogger('.'.join(__name__.split('.')[:-1]))
13
+
14
+
15
+ def _get_func_binding(
16
+ func,
17
+ exclude_args: List[str],
18
+ output_dirs: List[str],
19
+ args,
20
+ kwargs,
21
+ ):
22
+ """
23
+ Note: does not exclude exclude_args_if_default, which is fine since this is
24
+ used just for logging purposes and nothing else.
25
+ """
26
+ args = deepcopy(args)
27
+ kwargs = deepcopy(kwargs)
28
+ unhashed_args = exclude_args + output_dirs
29
+ for unhashed_arg in unhashed_args:
30
+ kwargs[unhashed_arg] = None
31
+ s = signature(func)
32
+ binding = s.bind(*args, **kwargs)
33
+ binding.apply_defaults()
34
+ return binding
35
+
36
+
37
+ def _get_func_caching_dir_aux(
38
+ func,
39
+ exclude_args: List[str],
40
+ exclude_args_if_default: List[str],
41
+ output_dirs: List[str],
42
+ args,
43
+ kwargs,
44
+ cache_dir: str,
45
+ use_hash: bool,
46
+ ) -> str:
47
+ """
48
+ Get caching directory for the given *function call*.
49
+ """
50
+ args = deepcopy(args)
51
+ kwargs = deepcopy(kwargs)
52
+
53
+ # To bind the func call, we need to have all arguments specified. Since the
54
+ # output dirs can be omitted, we just hackily assign them None. TODO: If
55
+ # the output dirs are passed as args, this fails due to a `TypeError:
56
+ # multiple values for argument`. The good news is that it fails noisily,
57
+ # instead of causing a silent bug. So really not critical to address right
58
+ # now.
59
+ for output_dir in output_dirs:
60
+ kwargs[output_dir] = None
61
+
62
+ unhashed_args = exclude_args + output_dirs
63
+ # Now add the exclude_args_if_default
64
+ s = signature(func)
65
+ binding = s.bind(*args, **kwargs)
66
+ binding.apply_defaults()
67
+ for (arg, val) in binding.arguments.items():
68
+ if arg in exclude_args_if_default:
69
+ p = s.parameters[arg]
70
+ default = p.default
71
+ if val == default:
72
+ logger.info(f"Excluding {arg} because it has default value {val}.")
73
+ unhashed_args.append(arg)
74
+
75
+ return _get_func_caching_dir(
76
+ func=func,
77
+ unhashed_args=unhashed_args,
78
+ args=args,
79
+ kwargs=kwargs,
80
+ cache_dir=cache_dir,
81
+ use_hash=use_hash,
82
+ )
83
+
84
+
85
+ def _maybe_write_usefull_stuff_cached_computation(
86
+ func,
87
+ exclude_args,
88
+ exclude_args_if_default,
89
+ output_dirs,
90
+ args,
91
+ kwargs,
92
+ cache_dir,
93
+ output_dir,
94
+ write_extra_log_files,
95
+ ):
96
+ """
97
+ Creates the logfiles _unhashed_output_dir.log and
98
+ _function_binding.log which contain useful information
99
+ about the exact function call (in case hashing was used
100
+ to compute the output directory).
101
+
102
+ The files are written if they already exist or their mode
103
+ is not 444.
104
+ """
105
+ if not write_extra_log_files:
106
+ return
107
+
108
+ unhashed_func_caching_dir = _get_func_caching_dir_aux(
109
+ func,
110
+ exclude_args,
111
+ exclude_args_if_default,
112
+ output_dirs,
113
+ args,
114
+ kwargs,
115
+ cache_dir,
116
+ use_hash=False,
117
+ )
118
+ unhashed_func_caching_dir_logfile = os.path.join(
119
+ kwargs[output_dir],
120
+ "_unhashed_output_dir.log",
121
+ )
122
+ if not (
123
+ os.path.exists(unhashed_func_caching_dir_logfile)
124
+ and _get_mode(unhashed_func_caching_dir_logfile) == "444"
125
+ ):
126
+ open(unhashed_func_caching_dir_logfile, "w").write(
127
+ unhashed_func_caching_dir
128
+ )
129
+ os.system(f'chmod 444 "{unhashed_func_caching_dir_logfile}"')
130
+
131
+ binding = _get_func_binding(
132
+ func,
133
+ exclude_args,
134
+ output_dirs,
135
+ args,
136
+ kwargs,
137
+ )
138
+ func_binding_logfile = os.path.join(
139
+ kwargs[output_dir],
140
+ "_function_binding.log",
141
+ )
142
+ if not (
143
+ os.path.exists(func_binding_logfile)
144
+ and _get_mode(func_binding_logfile) == "444"
145
+ ):
146
+ open(func_binding_logfile, "w").write(str(binding))
147
+ os.system(f'chmod 444 "{func_binding_logfile}"')
148
+
149
+
150
+ def cached_computation(
151
+ exclude_args: List = [],
152
+ exclude_args_if_default: List = [],
153
+ output_dirs: List = [],
154
+ write_extra_log_files: bool = False,
155
+ ):
156
+ """
157
+ Cache a function's outputs.
158
+
159
+ The function must write its outputs to `output_dirs[i]/result.txt` for each
160
+ i. The files `output_dirs[i]/result.txt` will be changed to mode 444 by the
161
+ decorator to avoid data corruption, and upon successful execution a file
162
+ `output_dirs[i]/result.success` will be created. This caching
163
+ scheme ensures (in two redundant ways) that data is never corrupted. If an
164
+ output file exists but is found to not have mode 444, or not have a
165
+ corresponding success token, then the output will be considered corrupted
166
+ and the computation will be performed.
167
+
168
+ The computation performed by the function should be a function of all
169
+ arguments to the function except for the provided `exclude_args`,
170
+ and the `output_dirs`. This tells the decorator
171
+ how to set the `output_dirs` to look for previously computed results.
172
+ The only argument in the decorator that has some wiggle room is the
173
+ `exclude_args`. Arguments that we usually want to specify here include
174
+ things like the total number of CPU cores to use; whether to use CPU or GPU,
175
+ how many OpenMP threads to use; how many MPI ranks to use; the verbosity
176
+ level. The latter are all arguments related to parallelization or logging
177
+ that the function does not depend on, so we exclude them. If they are not
178
+ excluded, then e.g. running the same function with more CPU cores will
179
+ result in all the computation being redone, which is ridiculous!
180
+
181
+ Because the output directories are determined automatically by the
182
+ decorator, there is not way to know what they are. Because of this, the
183
+ decorated function will *return the output directories* when called. This
184
+ allows seamlessly piecing together the computation performed by
185
+ cached parallel functions. The return value is, more precisely, a dictionary
186
+ mapping each element of `output_dirs` to its path.
187
+
188
+ Args:
189
+ exclude_args: Arguments which should be excluded when computing the
190
+ cache key. For example, the number of processors to use.
191
+ exclude_args_if_default: Arguments which should be excluded when
192
+ computing the cache key, ONLY IF THEY TAKE THEIR DEFAULT VALUE.
193
+ Good for backwards compatibility when extending the cached function
194
+ with new arguments.
195
+ output_dirs: The list of output directories of the wrapped function.
196
+ Each value in the argument specified by parallel_arg creates one
197
+ output in each directory specified by output_dirs.
198
+ write_extra_log_files: If to write extra log files indicating e.g. the
199
+ full function call. For functions that are called many times (e.g.
200
+ metrics) with many arguments, the extra log files can clog up space,
201
+ so it if useful to set this argument to `False`.
202
+
203
+ Returns:
204
+ The concrete caching decorator.
205
+ """
206
+
207
+ def cached_computation_decorator(func):
208
+ """
209
+ Concrete caching decorator.
210
+
211
+ The concrete decorator resulting from the provided exclude_args,
212
+ output_dirs.
213
+
214
+ Args:
215
+ func: The wrapped function.
216
+
217
+ Returns:
218
+ The wrapped function.
219
+ """
220
+ _validate_decorator_args(
221
+ func, decorator_args=exclude_args + exclude_args_if_default + output_dirs
222
+ )
223
+
224
+ @wraps(func)
225
+ def wrapper(*args, **kwargs):
226
+ # Only allow calling the function with kwargs for simplicity. TODO:
227
+ # If I find a way to put all the args into kwargs, I can remove
228
+ # this restriction from the user.
229
+ if len(args) > 0:
230
+ raise CacheUsageError(
231
+ f"Please call {func.__name__} with keyword arguments only. "
232
+ f"Positional arguments are not allowed for caching reasons."
233
+ )
234
+
235
+ # Get caching hyperparameters
236
+ cache_dir = get_cache_dir()
237
+ if cache_dir is None:
238
+ return func(*args, **kwargs)
239
+ use_hash = get_use_hash()
240
+
241
+ # Compute function caching directory.
242
+ func_caching_dir = _get_func_caching_dir_aux(
243
+ func,
244
+ exclude_args,
245
+ exclude_args_if_default,
246
+ output_dirs,
247
+ args,
248
+ kwargs,
249
+ cache_dir,
250
+ use_hash,
251
+ )
252
+
253
+ # Set the output dirs in kwargs based on the caching directory if
254
+ # not already provided.
255
+ for output_dir in output_dirs:
256
+ if output_dir not in kwargs or kwargs[output_dir] is None:
257
+ kwargs[output_dir] = os.path.join(
258
+ func_caching_dir, output_dir
259
+ )
260
+
261
+ # We will return the output directories. We get them right now in
262
+ # case these get modified by the call.
263
+ res = {output_dir: kwargs[output_dir] for output_dir in output_dirs}
264
+
265
+ def computed():
266
+ # Check that each of the output files exists
267
+ for output_dir in output_dirs:
268
+ output_filepath = os.path.join(
269
+ kwargs[output_dir], "result.txt"
270
+ )
271
+
272
+ if not os.path.exists(output_filepath):
273
+ return False
274
+
275
+ # COMMENTED OUT: Don't be so picky about the file mode.
276
+ # mode = _get_mode(output_filepath)
277
+ # if mode != "444":
278
+ # return False
279
+
280
+ output_success_token_filepath = os.path.join(
281
+ kwargs[output_dir], "result.success"
282
+ )
283
+ if not os.path.exists(output_success_token_filepath):
284
+ return False
285
+ return True
286
+
287
+ def clear_previous_outputs():
288
+ for output_dir in output_dirs:
289
+ output_filepath = os.path.join(
290
+ kwargs[output_dir], "result.txt"
291
+ )
292
+ if os.path.exists(output_filepath):
293
+ logger.info(
294
+ f"Removing possibly corrupted {output_filepath}"
295
+ )
296
+ os.system(f'chmod 666 "{output_filepath}"')
297
+ os.remove(output_filepath)
298
+
299
+ output_success_token_filepath = os.path.join(
300
+ kwargs[output_dir], "result.success"
301
+ )
302
+ if os.path.exists(output_success_token_filepath):
303
+ logger.info(f"Removing {output_success_token_filepath}")
304
+ os.system(
305
+ f'chmod 666 "{output_success_token_filepath}"'
306
+ )
307
+ os.remove(output_success_token_filepath)
308
+
309
+ # Make sure that all the output directories exist.
310
+ for output_dir in output_dirs:
311
+ if not os.path.exists(kwargs[output_dir]):
312
+ os.umask(0) # To ensure all collaborators can access cache
313
+ os.makedirs(kwargs[output_dir], mode=0o777)
314
+ # Let's write some useful stuff to the directory
315
+ _maybe_write_usefull_stuff_cached_computation(
316
+ func,
317
+ exclude_args,
318
+ exclude_args_if_default,
319
+ output_dirs,
320
+ args,
321
+ kwargs,
322
+ cache_dir,
323
+ output_dir,
324
+ write_extra_log_files,
325
+ )
326
+
327
+ # Only call the function if there is any work to do at all.
328
+ if not computed():
329
+ # Now call the wrapped function
330
+ logger.debug(
331
+ f"Calling {func.__name__} . Output location: {func_caching_dir}"
332
+ )
333
+ if get_read_only():
334
+ raise CacheUsageError(
335
+ "Cache is in read only mode! Will not call function."
336
+ )
337
+ clear_previous_outputs()
338
+ func(*args, **kwargs)
339
+
340
+ # Now verify that all outputs are there.
341
+ for output_dir in output_dirs:
342
+ output_filepath = os.path.join(
343
+ kwargs[output_dir], "result.txt"
344
+ )
345
+ if not os.path.exists(output_filepath):
346
+ raise CacheUsageError(
347
+ f"function {func.__name__} should have created "
348
+ f"and written output to {output_filepath} but "
349
+ "the file does not exist."
350
+ )
351
+
352
+ # Now chmod 444 all the output files and create the success
353
+ # tokens for as redundant verification.
354
+ for output_dir in output_dirs:
355
+ output_filepath = os.path.join(
356
+ kwargs[output_dir], "result.txt"
357
+ )
358
+ os.system(f'chmod 444 "{output_filepath}"')
359
+ output_success_token_filepath = os.path.join(
360
+ kwargs[output_dir], "result.success"
361
+ )
362
+ with open(output_success_token_filepath, "w") as f:
363
+ f.write("SUCCESS\n")
364
+ f.flush()
365
+ return res
366
+
367
+ return wrapper
368
+
369
+ return cached_computation_decorator