nova-clone 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nova_clone/__init__.py +1 -0
- nova_clone/algorithm/__init__.py +2 -0
- nova_clone/algorithm/_build_intbc_graph.py +135 -0
- nova_clone/algorithm/_call_clones_from_coresets.py +235 -0
- nova_clone/algorithm/_clone_call_ops.py +258 -0
- nova_clone/algorithm/_clone_calling_algorithm.py +1571 -0
- nova_clone/algorithm/_compute_cellbc_vs_coreset_overlap_matrix.py +143 -0
- nova_clone/algorithm/_compute_coreset_co_assignment_matrix.py +170 -0
- nova_clone/algorithm/_find_maximal_coresets.py +428 -0
- nova_clone/algorithm/_keep_reliable_edges.py +202 -0
- nova_clone/algorithm/_make_coresets_disjoint.py +114 -0
- nova_clone/algorithm/_prune_coresets.py +164 -0
- nova_clone/algorithm/_prune_custom_intbcs.py +150 -0
- nova_clone/algorithm/_prune_likely_homoplastic_intbcs.py +162 -0
- nova_clone/algorithm/_prune_likely_sequencing_errors.py +201 -0
- nova_clone/algorithm/_prune_low_cellbc_count_intbcs.py +154 -0
- nova_clone/algorithm/_prune_low_umi_count_intbcs.py +152 -0
- nova_clone/algorithm/_types.py +150 -0
- nova_clone/algorithm/_utils.py +53 -0
- nova_clone/algorithm/cassiopeia/__init__.py +1 -0
- nova_clone/algorithm/cassiopeia/_call_lineage_groups.py +270 -0
- nova_clone/caching/__init__.py +24 -0
- nova_clone/caching/_cached.py +175 -0
- nova_clone/caching/_cached_computation.py +369 -0
- nova_clone/caching/_cached_parallel_computation.py +428 -0
- nova_clone/caching/_common.py +200 -0
- nova_clone/data/__init__.py +1 -0
- nova_clone/qc/__init__.py +179 -0
- nova_clone/qc/_clone_intbc_sets.py +234 -0
- nova_clone/qc/_compare_clone_calls.py +218 -0
- nova_clone/qc/_data_retention.py +175 -0
- nova_clone/qc/_homoplasy.py +81 -0
- nova_clone/qc/_inter_clonal_kinship.py +60 -0
- nova_clone/qc/_intra_clonal_kinship.py +172 -0
- nova_clone/qc/_multiplet_rate.py +158 -0
- nova_clone/qc/_number_of_clones.py +153 -0
- nova_clone/qc/_pre_qc.py +485 -0
- nova_clone/simulation/HDBSCAN_method_adjusted.py +68 -0
- nova_clone/simulation/__init__.py +0 -0
- nova_clone/simulation/_caching.py +40 -0
- nova_clone/simulation/benchmarking.py +1221 -0
- nova_clone/simulation/clone_to_sequences.py +408 -0
- nova_clone/simulation/creating_tree.py +416 -0
- nova_clone/simulation/evaluation.py +344 -0
- nova_clone/simulation/jaccard_hierarchical_by_intBC.py +78 -0
- nova_clone/simulation/plot_benchmark_lines.py +226 -0
- nova_clone/simulation/slice_benchmark.py +640 -0
- nova_clone/simulation/slice_benchmark_clone_calling.py +68 -0
- nova_clone-0.1.0.dist-info/METADATA +298 -0
- nova_clone-0.1.0.dist-info/RECORD +53 -0
- nova_clone-0.1.0.dist-info/WHEEL +5 -0
- nova_clone-0.1.0.dist-info/licenses/LICENSE +28 -0
- nova_clone-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
from copy import deepcopy
|
|
4
|
+
from functools import wraps
|
|
5
|
+
from inspect import signature
|
|
6
|
+
from typing import List
|
|
7
|
+
|
|
8
|
+
from ._common import (CacheUsageError, _get_func_caching_dir, _get_mode,
|
|
9
|
+
_validate_decorator_args, get_cache_dir, get_read_only,
|
|
10
|
+
get_use_hash)
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger('.'.join(__name__.split('.')[:-1]))
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _get_func_binding(
|
|
16
|
+
func,
|
|
17
|
+
exclude_args: List[str],
|
|
18
|
+
output_dirs: List[str],
|
|
19
|
+
args,
|
|
20
|
+
kwargs,
|
|
21
|
+
):
|
|
22
|
+
"""
|
|
23
|
+
Note: does not exclude exclude_args_if_default, which is fine since this is
|
|
24
|
+
used just for logging purposes and nothing else.
|
|
25
|
+
"""
|
|
26
|
+
args = deepcopy(args)
|
|
27
|
+
kwargs = deepcopy(kwargs)
|
|
28
|
+
unhashed_args = exclude_args + output_dirs
|
|
29
|
+
for unhashed_arg in unhashed_args:
|
|
30
|
+
kwargs[unhashed_arg] = None
|
|
31
|
+
s = signature(func)
|
|
32
|
+
binding = s.bind(*args, **kwargs)
|
|
33
|
+
binding.apply_defaults()
|
|
34
|
+
return binding
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _get_func_caching_dir_aux(
|
|
38
|
+
func,
|
|
39
|
+
exclude_args: List[str],
|
|
40
|
+
exclude_args_if_default: List[str],
|
|
41
|
+
output_dirs: List[str],
|
|
42
|
+
args,
|
|
43
|
+
kwargs,
|
|
44
|
+
cache_dir: str,
|
|
45
|
+
use_hash: bool,
|
|
46
|
+
) -> str:
|
|
47
|
+
"""
|
|
48
|
+
Get caching directory for the given *function call*.
|
|
49
|
+
"""
|
|
50
|
+
args = deepcopy(args)
|
|
51
|
+
kwargs = deepcopy(kwargs)
|
|
52
|
+
|
|
53
|
+
# To bind the func call, we need to have all arguments specified. Since the
|
|
54
|
+
# output dirs can be omitted, we just hackily assign them None. TODO: If
|
|
55
|
+
# the output dirs are passed as args, this fails due to a `TypeError:
|
|
56
|
+
# multiple values for argument`. The good news is that it fails noisily,
|
|
57
|
+
# instead of causing a silent bug. So really not critical to address right
|
|
58
|
+
# now.
|
|
59
|
+
for output_dir in output_dirs:
|
|
60
|
+
kwargs[output_dir] = None
|
|
61
|
+
|
|
62
|
+
unhashed_args = exclude_args + output_dirs
|
|
63
|
+
# Now add the exclude_args_if_default
|
|
64
|
+
s = signature(func)
|
|
65
|
+
binding = s.bind(*args, **kwargs)
|
|
66
|
+
binding.apply_defaults()
|
|
67
|
+
for (arg, val) in binding.arguments.items():
|
|
68
|
+
if arg in exclude_args_if_default:
|
|
69
|
+
p = s.parameters[arg]
|
|
70
|
+
default = p.default
|
|
71
|
+
if val == default:
|
|
72
|
+
logger.info(f"Excluding {arg} because it has default value {val}.")
|
|
73
|
+
unhashed_args.append(arg)
|
|
74
|
+
|
|
75
|
+
return _get_func_caching_dir(
|
|
76
|
+
func=func,
|
|
77
|
+
unhashed_args=unhashed_args,
|
|
78
|
+
args=args,
|
|
79
|
+
kwargs=kwargs,
|
|
80
|
+
cache_dir=cache_dir,
|
|
81
|
+
use_hash=use_hash,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _maybe_write_usefull_stuff_cached_computation(
|
|
86
|
+
func,
|
|
87
|
+
exclude_args,
|
|
88
|
+
exclude_args_if_default,
|
|
89
|
+
output_dirs,
|
|
90
|
+
args,
|
|
91
|
+
kwargs,
|
|
92
|
+
cache_dir,
|
|
93
|
+
output_dir,
|
|
94
|
+
write_extra_log_files,
|
|
95
|
+
):
|
|
96
|
+
"""
|
|
97
|
+
Creates the logfiles _unhashed_output_dir.log and
|
|
98
|
+
_function_binding.log which contain useful information
|
|
99
|
+
about the exact function call (in case hashing was used
|
|
100
|
+
to compute the output directory).
|
|
101
|
+
|
|
102
|
+
The files are written if they already exist or their mode
|
|
103
|
+
is not 444.
|
|
104
|
+
"""
|
|
105
|
+
if not write_extra_log_files:
|
|
106
|
+
return
|
|
107
|
+
|
|
108
|
+
unhashed_func_caching_dir = _get_func_caching_dir_aux(
|
|
109
|
+
func,
|
|
110
|
+
exclude_args,
|
|
111
|
+
exclude_args_if_default,
|
|
112
|
+
output_dirs,
|
|
113
|
+
args,
|
|
114
|
+
kwargs,
|
|
115
|
+
cache_dir,
|
|
116
|
+
use_hash=False,
|
|
117
|
+
)
|
|
118
|
+
unhashed_func_caching_dir_logfile = os.path.join(
|
|
119
|
+
kwargs[output_dir],
|
|
120
|
+
"_unhashed_output_dir.log",
|
|
121
|
+
)
|
|
122
|
+
if not (
|
|
123
|
+
os.path.exists(unhashed_func_caching_dir_logfile)
|
|
124
|
+
and _get_mode(unhashed_func_caching_dir_logfile) == "444"
|
|
125
|
+
):
|
|
126
|
+
open(unhashed_func_caching_dir_logfile, "w").write(
|
|
127
|
+
unhashed_func_caching_dir
|
|
128
|
+
)
|
|
129
|
+
os.system(f'chmod 444 "{unhashed_func_caching_dir_logfile}"')
|
|
130
|
+
|
|
131
|
+
binding = _get_func_binding(
|
|
132
|
+
func,
|
|
133
|
+
exclude_args,
|
|
134
|
+
output_dirs,
|
|
135
|
+
args,
|
|
136
|
+
kwargs,
|
|
137
|
+
)
|
|
138
|
+
func_binding_logfile = os.path.join(
|
|
139
|
+
kwargs[output_dir],
|
|
140
|
+
"_function_binding.log",
|
|
141
|
+
)
|
|
142
|
+
if not (
|
|
143
|
+
os.path.exists(func_binding_logfile)
|
|
144
|
+
and _get_mode(func_binding_logfile) == "444"
|
|
145
|
+
):
|
|
146
|
+
open(func_binding_logfile, "w").write(str(binding))
|
|
147
|
+
os.system(f'chmod 444 "{func_binding_logfile}"')
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def cached_computation(
|
|
151
|
+
exclude_args: List = [],
|
|
152
|
+
exclude_args_if_default: List = [],
|
|
153
|
+
output_dirs: List = [],
|
|
154
|
+
write_extra_log_files: bool = False,
|
|
155
|
+
):
|
|
156
|
+
"""
|
|
157
|
+
Cache a function's outputs.
|
|
158
|
+
|
|
159
|
+
The function must write its outputs to `output_dirs[i]/result.txt` for each
|
|
160
|
+
i. The files `output_dirs[i]/result.txt` will be changed to mode 444 by the
|
|
161
|
+
decorator to avoid data corruption, and upon successful execution a file
|
|
162
|
+
`output_dirs[i]/result.success` will be created. This caching
|
|
163
|
+
scheme ensures (in two redundant ways) that data is never corrupted. If an
|
|
164
|
+
output file exists but is found to not have mode 444, or not have a
|
|
165
|
+
corresponding success token, then the output will be considered corrupted
|
|
166
|
+
and the computation will be performed.
|
|
167
|
+
|
|
168
|
+
The computation performed by the function should be a function of all
|
|
169
|
+
arguments to the function except for the provided `exclude_args`,
|
|
170
|
+
and the `output_dirs`. This tells the decorator
|
|
171
|
+
how to set the `output_dirs` to look for previously computed results.
|
|
172
|
+
The only argument in the decorator that has some wiggle room is the
|
|
173
|
+
`exclude_args`. Arguments that we usually want to specify here include
|
|
174
|
+
things like the total number of CPU cores to use; whether to use CPU or GPU,
|
|
175
|
+
how many OpenMP threads to use; how many MPI ranks to use; the verbosity
|
|
176
|
+
level. The latter are all arguments related to parallelization or logging
|
|
177
|
+
that the function does not depend on, so we exclude them. If they are not
|
|
178
|
+
excluded, then e.g. running the same function with more CPU cores will
|
|
179
|
+
result in all the computation being redone, which is ridiculous!
|
|
180
|
+
|
|
181
|
+
Because the output directories are determined automatically by the
|
|
182
|
+
decorator, there is not way to know what they are. Because of this, the
|
|
183
|
+
decorated function will *return the output directories* when called. This
|
|
184
|
+
allows seamlessly piecing together the computation performed by
|
|
185
|
+
cached parallel functions. The return value is, more precisely, a dictionary
|
|
186
|
+
mapping each element of `output_dirs` to its path.
|
|
187
|
+
|
|
188
|
+
Args:
|
|
189
|
+
exclude_args: Arguments which should be excluded when computing the
|
|
190
|
+
cache key. For example, the number of processors to use.
|
|
191
|
+
exclude_args_if_default: Arguments which should be excluded when
|
|
192
|
+
computing the cache key, ONLY IF THEY TAKE THEIR DEFAULT VALUE.
|
|
193
|
+
Good for backwards compatibility when extending the cached function
|
|
194
|
+
with new arguments.
|
|
195
|
+
output_dirs: The list of output directories of the wrapped function.
|
|
196
|
+
Each value in the argument specified by parallel_arg creates one
|
|
197
|
+
output in each directory specified by output_dirs.
|
|
198
|
+
write_extra_log_files: If to write extra log files indicating e.g. the
|
|
199
|
+
full function call. For functions that are called many times (e.g.
|
|
200
|
+
metrics) with many arguments, the extra log files can clog up space,
|
|
201
|
+
so it if useful to set this argument to `False`.
|
|
202
|
+
|
|
203
|
+
Returns:
|
|
204
|
+
The concrete caching decorator.
|
|
205
|
+
"""
|
|
206
|
+
|
|
207
|
+
def cached_computation_decorator(func):
|
|
208
|
+
"""
|
|
209
|
+
Concrete caching decorator.
|
|
210
|
+
|
|
211
|
+
The concrete decorator resulting from the provided exclude_args,
|
|
212
|
+
output_dirs.
|
|
213
|
+
|
|
214
|
+
Args:
|
|
215
|
+
func: The wrapped function.
|
|
216
|
+
|
|
217
|
+
Returns:
|
|
218
|
+
The wrapped function.
|
|
219
|
+
"""
|
|
220
|
+
_validate_decorator_args(
|
|
221
|
+
func, decorator_args=exclude_args + exclude_args_if_default + output_dirs
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
@wraps(func)
|
|
225
|
+
def wrapper(*args, **kwargs):
|
|
226
|
+
# Only allow calling the function with kwargs for simplicity. TODO:
|
|
227
|
+
# If I find a way to put all the args into kwargs, I can remove
|
|
228
|
+
# this restriction from the user.
|
|
229
|
+
if len(args) > 0:
|
|
230
|
+
raise CacheUsageError(
|
|
231
|
+
f"Please call {func.__name__} with keyword arguments only. "
|
|
232
|
+
f"Positional arguments are not allowed for caching reasons."
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
# Get caching hyperparameters
|
|
236
|
+
cache_dir = get_cache_dir()
|
|
237
|
+
if cache_dir is None:
|
|
238
|
+
return func(*args, **kwargs)
|
|
239
|
+
use_hash = get_use_hash()
|
|
240
|
+
|
|
241
|
+
# Compute function caching directory.
|
|
242
|
+
func_caching_dir = _get_func_caching_dir_aux(
|
|
243
|
+
func,
|
|
244
|
+
exclude_args,
|
|
245
|
+
exclude_args_if_default,
|
|
246
|
+
output_dirs,
|
|
247
|
+
args,
|
|
248
|
+
kwargs,
|
|
249
|
+
cache_dir,
|
|
250
|
+
use_hash,
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
# Set the output dirs in kwargs based on the caching directory if
|
|
254
|
+
# not already provided.
|
|
255
|
+
for output_dir in output_dirs:
|
|
256
|
+
if output_dir not in kwargs or kwargs[output_dir] is None:
|
|
257
|
+
kwargs[output_dir] = os.path.join(
|
|
258
|
+
func_caching_dir, output_dir
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
# We will return the output directories. We get them right now in
|
|
262
|
+
# case these get modified by the call.
|
|
263
|
+
res = {output_dir: kwargs[output_dir] for output_dir in output_dirs}
|
|
264
|
+
|
|
265
|
+
def computed():
|
|
266
|
+
# Check that each of the output files exists
|
|
267
|
+
for output_dir in output_dirs:
|
|
268
|
+
output_filepath = os.path.join(
|
|
269
|
+
kwargs[output_dir], "result.txt"
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
if not os.path.exists(output_filepath):
|
|
273
|
+
return False
|
|
274
|
+
|
|
275
|
+
# COMMENTED OUT: Don't be so picky about the file mode.
|
|
276
|
+
# mode = _get_mode(output_filepath)
|
|
277
|
+
# if mode != "444":
|
|
278
|
+
# return False
|
|
279
|
+
|
|
280
|
+
output_success_token_filepath = os.path.join(
|
|
281
|
+
kwargs[output_dir], "result.success"
|
|
282
|
+
)
|
|
283
|
+
if not os.path.exists(output_success_token_filepath):
|
|
284
|
+
return False
|
|
285
|
+
return True
|
|
286
|
+
|
|
287
|
+
def clear_previous_outputs():
|
|
288
|
+
for output_dir in output_dirs:
|
|
289
|
+
output_filepath = os.path.join(
|
|
290
|
+
kwargs[output_dir], "result.txt"
|
|
291
|
+
)
|
|
292
|
+
if os.path.exists(output_filepath):
|
|
293
|
+
logger.info(
|
|
294
|
+
f"Removing possibly corrupted {output_filepath}"
|
|
295
|
+
)
|
|
296
|
+
os.system(f'chmod 666 "{output_filepath}"')
|
|
297
|
+
os.remove(output_filepath)
|
|
298
|
+
|
|
299
|
+
output_success_token_filepath = os.path.join(
|
|
300
|
+
kwargs[output_dir], "result.success"
|
|
301
|
+
)
|
|
302
|
+
if os.path.exists(output_success_token_filepath):
|
|
303
|
+
logger.info(f"Removing {output_success_token_filepath}")
|
|
304
|
+
os.system(
|
|
305
|
+
f'chmod 666 "{output_success_token_filepath}"'
|
|
306
|
+
)
|
|
307
|
+
os.remove(output_success_token_filepath)
|
|
308
|
+
|
|
309
|
+
# Make sure that all the output directories exist.
|
|
310
|
+
for output_dir in output_dirs:
|
|
311
|
+
if not os.path.exists(kwargs[output_dir]):
|
|
312
|
+
os.umask(0) # To ensure all collaborators can access cache
|
|
313
|
+
os.makedirs(kwargs[output_dir], mode=0o777)
|
|
314
|
+
# Let's write some useful stuff to the directory
|
|
315
|
+
_maybe_write_usefull_stuff_cached_computation(
|
|
316
|
+
func,
|
|
317
|
+
exclude_args,
|
|
318
|
+
exclude_args_if_default,
|
|
319
|
+
output_dirs,
|
|
320
|
+
args,
|
|
321
|
+
kwargs,
|
|
322
|
+
cache_dir,
|
|
323
|
+
output_dir,
|
|
324
|
+
write_extra_log_files,
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
# Only call the function if there is any work to do at all.
|
|
328
|
+
if not computed():
|
|
329
|
+
# Now call the wrapped function
|
|
330
|
+
logger.debug(
|
|
331
|
+
f"Calling {func.__name__} . Output location: {func_caching_dir}"
|
|
332
|
+
)
|
|
333
|
+
if get_read_only():
|
|
334
|
+
raise CacheUsageError(
|
|
335
|
+
"Cache is in read only mode! Will not call function."
|
|
336
|
+
)
|
|
337
|
+
clear_previous_outputs()
|
|
338
|
+
func(*args, **kwargs)
|
|
339
|
+
|
|
340
|
+
# Now verify that all outputs are there.
|
|
341
|
+
for output_dir in output_dirs:
|
|
342
|
+
output_filepath = os.path.join(
|
|
343
|
+
kwargs[output_dir], "result.txt"
|
|
344
|
+
)
|
|
345
|
+
if not os.path.exists(output_filepath):
|
|
346
|
+
raise CacheUsageError(
|
|
347
|
+
f"function {func.__name__} should have created "
|
|
348
|
+
f"and written output to {output_filepath} but "
|
|
349
|
+
"the file does not exist."
|
|
350
|
+
)
|
|
351
|
+
|
|
352
|
+
# Now chmod 444 all the output files and create the success
|
|
353
|
+
# tokens for as redundant verification.
|
|
354
|
+
for output_dir in output_dirs:
|
|
355
|
+
output_filepath = os.path.join(
|
|
356
|
+
kwargs[output_dir], "result.txt"
|
|
357
|
+
)
|
|
358
|
+
os.system(f'chmod 444 "{output_filepath}"')
|
|
359
|
+
output_success_token_filepath = os.path.join(
|
|
360
|
+
kwargs[output_dir], "result.success"
|
|
361
|
+
)
|
|
362
|
+
with open(output_success_token_filepath, "w") as f:
|
|
363
|
+
f.write("SUCCESS\n")
|
|
364
|
+
f.flush()
|
|
365
|
+
return res
|
|
366
|
+
|
|
367
|
+
return wrapper
|
|
368
|
+
|
|
369
|
+
return cached_computation_decorator
|