nova-clone 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. nova_clone/__init__.py +1 -0
  2. nova_clone/algorithm/__init__.py +2 -0
  3. nova_clone/algorithm/_build_intbc_graph.py +135 -0
  4. nova_clone/algorithm/_call_clones_from_coresets.py +235 -0
  5. nova_clone/algorithm/_clone_call_ops.py +258 -0
  6. nova_clone/algorithm/_clone_calling_algorithm.py +1571 -0
  7. nova_clone/algorithm/_compute_cellbc_vs_coreset_overlap_matrix.py +143 -0
  8. nova_clone/algorithm/_compute_coreset_co_assignment_matrix.py +170 -0
  9. nova_clone/algorithm/_find_maximal_coresets.py +428 -0
  10. nova_clone/algorithm/_keep_reliable_edges.py +202 -0
  11. nova_clone/algorithm/_make_coresets_disjoint.py +114 -0
  12. nova_clone/algorithm/_prune_coresets.py +164 -0
  13. nova_clone/algorithm/_prune_custom_intbcs.py +150 -0
  14. nova_clone/algorithm/_prune_likely_homoplastic_intbcs.py +162 -0
  15. nova_clone/algorithm/_prune_likely_sequencing_errors.py +201 -0
  16. nova_clone/algorithm/_prune_low_cellbc_count_intbcs.py +154 -0
  17. nova_clone/algorithm/_prune_low_umi_count_intbcs.py +152 -0
  18. nova_clone/algorithm/_types.py +150 -0
  19. nova_clone/algorithm/_utils.py +53 -0
  20. nova_clone/algorithm/cassiopeia/__init__.py +1 -0
  21. nova_clone/algorithm/cassiopeia/_call_lineage_groups.py +270 -0
  22. nova_clone/caching/__init__.py +24 -0
  23. nova_clone/caching/_cached.py +175 -0
  24. nova_clone/caching/_cached_computation.py +369 -0
  25. nova_clone/caching/_cached_parallel_computation.py +428 -0
  26. nova_clone/caching/_common.py +200 -0
  27. nova_clone/data/__init__.py +1 -0
  28. nova_clone/qc/__init__.py +179 -0
  29. nova_clone/qc/_clone_intbc_sets.py +234 -0
  30. nova_clone/qc/_compare_clone_calls.py +218 -0
  31. nova_clone/qc/_data_retention.py +175 -0
  32. nova_clone/qc/_homoplasy.py +81 -0
  33. nova_clone/qc/_inter_clonal_kinship.py +60 -0
  34. nova_clone/qc/_intra_clonal_kinship.py +172 -0
  35. nova_clone/qc/_multiplet_rate.py +158 -0
  36. nova_clone/qc/_number_of_clones.py +153 -0
  37. nova_clone/qc/_pre_qc.py +485 -0
  38. nova_clone/simulation/HDBSCAN_method_adjusted.py +68 -0
  39. nova_clone/simulation/__init__.py +0 -0
  40. nova_clone/simulation/_caching.py +40 -0
  41. nova_clone/simulation/benchmarking.py +1221 -0
  42. nova_clone/simulation/clone_to_sequences.py +408 -0
  43. nova_clone/simulation/creating_tree.py +416 -0
  44. nova_clone/simulation/evaluation.py +344 -0
  45. nova_clone/simulation/jaccard_hierarchical_by_intBC.py +78 -0
  46. nova_clone/simulation/plot_benchmark_lines.py +226 -0
  47. nova_clone/simulation/slice_benchmark.py +640 -0
  48. nova_clone/simulation/slice_benchmark_clone_calling.py +68 -0
  49. nova_clone-0.1.0.dist-info/METADATA +298 -0
  50. nova_clone-0.1.0.dist-info/RECORD +53 -0
  51. nova_clone-0.1.0.dist-info/WHEEL +5 -0
  52. nova_clone-0.1.0.dist-info/licenses/LICENSE +28 -0
  53. nova_clone-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,53 @@
1
+ from typing import Set
2
+
3
+ from ._types import *
4
+
5
+
6
+ def _cell_intbc_sets_with_counts__for_intbc_table(
7
+ intbc_table: IntBCTableType,
8
+ ) -> Dict[CellBCType, Dict[IntBCType, UMICountType]]:
9
+ """
10
+ Get the intBC sets corresponding to each cell, w/#UMIs.
11
+ """
12
+ validate_intbc_table(intbc_table)
13
+ cell_intbc_sets_with_counts = {}
14
+ for cellbc, cell_df in intbc_table.groupby(
15
+ "cellBC"
16
+ ): # Not using tqdm bc its fast (<10 sec)
17
+ # Lets see what intBCs this cell has
18
+ intbc_and_umi_count = list(zip(list(cell_df.intBC), list(cell_df.UMI_count)))
19
+ cell_intbc_sets_with_counts[cellbc] = dict(intbc_and_umi_count)
20
+ return cell_intbc_sets_with_counts
21
+
22
+
23
+ def _cell_intbc_set_from_intbc_table(
24
+ intbc_table: IntBCTableType,
25
+ ) -> Dict[str, Set[str]]:
26
+ """
27
+ Get the intBC sets corresponding to each cell.
28
+ """
29
+ cell_intbc_sets_with_counts = _cell_intbc_sets_with_counts__for_intbc_table(
30
+ intbc_table=intbc_table,
31
+ )
32
+ cell_intbc_sets = {
33
+ cell: {intbc for (intbc, counts) in intbcs_with_counts.items()}
34
+ for (cell, intbcs_with_counts) in cell_intbc_sets_with_counts.items()
35
+ }
36
+ return cell_intbc_sets
37
+
38
+
39
+ def _cell_intbc_sets_from_intbc_tables(
40
+ intbc_table_per_sample: Dict[SampleNameType, IntBCTableType],
41
+ ) -> Dict[str, Set[str]]:
42
+ """
43
+ Get the intBC sets corresponding to each cell.
44
+ """
45
+ res = {}
46
+ for sample_name, intbc_table in intbc_table_per_sample.items():
47
+ cell_intbc_sets = _cell_intbc_set_from_intbc_table(
48
+ intbc_table=intbc_table,
49
+ )
50
+ for x, y in cell_intbc_sets.items():
51
+ assert x not in res
52
+ res[f"{sample_name}_" + x] = y
53
+ return res
@@ -0,0 +1 @@
1
+ from ._call_lineage_groups import call_lineage_groups
@@ -0,0 +1,270 @@
1
+ from copy import deepcopy
2
+
3
+ from cassiopeia.preprocess.pipeline import (
4
+ call_lineage_groups as call_lineage_groups_cassiopeia,
5
+ )
6
+ from nova_clone.algorithm._prune_low_umi_count_intbcs import prune_low_umi_count_intbcs
7
+
8
+ from nova_clone.algorithm._prune_low_umi_count_intbcs import (
9
+ prune_low_umi_count_intbcs,
10
+ )
11
+
12
+ from .._clone_calling_algorithm import _check_are_clones
13
+ from .._types import *
14
+
15
+
16
+ def _build_input_df(intbc_table_per_sample) -> pd.DataFrame:
17
+ rows = []
18
+ for sample_name, intbc_table in intbc_table_per_sample.items():
19
+ for _, row in intbc_table.iterrows():
20
+ row_dict = row.to_dict()
21
+ res = {
22
+ "cellBC": f"{sample_name}_{row_dict['cellBC']}",
23
+ "UMI": row_dict["UMI_count"],
24
+ "intBC": row_dict["intBC"],
25
+ "allele": "A",
26
+ "readCount": 1000,
27
+ }
28
+ rows.append(res)
29
+ return pd.DataFrame(rows)
30
+
31
+
32
+ def call_lineage_groups(
33
+ intbc_table_per_sample: Dict[SampleNameType, IntBCTableType],
34
+ min_umi_per_cell: int = 10,
35
+ min_avg_reads_per_umi: float = 2.0,
36
+ min_cluster_prop: float = 0.005,
37
+ min_intbc_thresh: float = 0.05,
38
+ inter_doublet_threshold: float = 0.35,
39
+ kinship_thresh: float = 0.25,
40
+ plot: bool = False,
41
+ min_intbc_umi_count: int = 0, # Default value is 0 because it is not used in the cassiopeia code. This is expected to be part of user preprocessing.
42
+ ) -> CloneCallType:
43
+ """
44
+ Wrapper around Cassiopeia's call_lineage_groups function.
45
+
46
+ Assigns cells to their clonal populations.
47
+
48
+ Performs multiple rounds of filtering and assigning to lineage groups:
49
+ 1. Iteratively generates putative lineage groups by forming intBC
50
+ groups for each lineage group and then assigning cells based on how
51
+ many intBCs they share with each intBC group (kinship).
52
+
53
+ 2. Refines these putative groups by removing non-informative intBCs
54
+ and reassigning cells through kinship.
55
+
56
+ 3. Removes all inter-lineage doublets, defined as cells that have
57
+ relatively equal kinship scores across multiple lineages and whose
58
+ assignments are therefore ambigious.
59
+
60
+ 4. Finally, performs one more round of filtering non-informative intBCs
61
+ and cellBCs with low UMI counts before returning a final table of
62
+ lineage assignments, allele information, and read and umi counts for
63
+ each sample.
64
+
65
+ Args:
66
+ intbc_table_per_sample: The intBC table per sample.
67
+ min_umi_per_cell: The threshold specifying the minimum number of UMIs a
68
+ cell needs in order to not be filtered during filtering
69
+ min_avg_reads_per_umi: The threshold specifying the minimum coverage
70
+ (i.e. average) reads per UMI in a cell needed in order for that
71
+ cell not to be filtered during filtering
72
+ min_cluster_prop: The minimum cluster size in the putative lineage
73
+ assignment step, as a proportion of the number of cells
74
+ min_intbc_thresh: The threshold specifying the minimum proportion of
75
+ cells in a lineage group that need to have an intBC in order for it
76
+ be retained during filtering. Also specifies the minimum proportion
77
+ of cells that share an intBC with the most frequent intBC in
78
+ forming putative lineage groups
79
+ inter_doublet_threshold: The threshold specifying the minimum proportion
80
+ of kinship a cell shares with its assigned lineage group out of all
81
+ lineage groups for it to be retained during doublet filtering
82
+ kinship_thresh: The threshold specifying the minimum proportion of
83
+ intBCs shared between a cell and the intBC set of a lineage group
84
+ needed to assign that cell to that lineage group in putative
85
+ assignment
86
+ plot: Indicates whether to generate plots. If so, they will be written to
87
+ `output_directory_cassiopeia`.
88
+ min_intbc_umi_count: The minimum number of UMIs for an intBC to be considered
89
+ in a cellBC.
90
+
91
+ Returns:
92
+ The clone call, with fields `clone_to_cells_mapping, cell_to_clones_mapping, clone_intbc_sets, cell_intbc_sets`.
93
+ """
94
+ validate_intbc_table_per_sample(intbc_table_per_sample=intbc_table_per_sample)
95
+ original_intbc_table_per_sample = deepcopy(intbc_table_per_sample)
96
+ intbc_table_per_sample = deepcopy(intbc_table_per_sample)
97
+
98
+ # Filter intBCs based on min_intbc_umi_count
99
+ intbc_table_per_sample = prune_low_umi_count_intbcs(
100
+ intbc_table_per_sample=intbc_table_per_sample,
101
+ min_intbc_umi_count=min_intbc_umi_count,
102
+ ).intbc_table_per_sample
103
+
104
+ input_df = _build_input_df(intbc_table_per_sample)
105
+ lineage_groups = call_lineage_groups_cassiopeia(
106
+ input_df=input_df,
107
+ output_directory="output_directory_cassiopeia",
108
+ min_umi_per_cell=min_umi_per_cell,
109
+ min_avg_reads_per_umi=min_avg_reads_per_umi,
110
+ min_cluster_prop=min_cluster_prop,
111
+ min_intbc_thresh=min_intbc_thresh,
112
+ inter_doublet_threshold=inter_doublet_threshold,
113
+ kinship_thresh=kinship_thresh,
114
+ plot=plot,
115
+ )
116
+
117
+ cell_to_clones_mapping = {}
118
+ cell_intbc_sets = {}
119
+ clone_intbc_sets = defaultdict(set)
120
+ for _, row in lineage_groups.iterrows():
121
+ row = row.to_dict()
122
+ clone = "clone_" + str(row["lineageGrp"])
123
+ cell = str(row["cellBC"])
124
+ if cell not in cell_to_clones_mapping:
125
+ cell_to_clones_mapping[cell] = set()
126
+ cell_to_clones_mapping[cell].add(clone)
127
+ clone_intbc_sets[clone].add(row["intBC"])
128
+ if cell not in cell_intbc_sets:
129
+ cell_intbc_sets[cell] = set()
130
+ cell_intbc_sets[cell].add(row["intBC"])
131
+ # Add cellBCs that were not assigned to any clone
132
+ for sample, intbc_table in original_intbc_table_per_sample.items():
133
+ for _, row in intbc_table.iterrows():
134
+ row = row.to_dict()
135
+ cellbc = f"{sample}_{row['cellBC']}"
136
+ if cellbc not in cell_to_clones_mapping:
137
+ cell_to_clones_mapping[cellbc] = set()
138
+ clone_to_cells_mapping = defaultdict(set)
139
+ for cell, clones in cell_to_clones_mapping.items():
140
+ for clone in clones:
141
+ clone_to_cells_mapping[clone].add(cell)
142
+ clone_intbc_sets = {x: y for (x, y) in clone_intbc_sets.items()}
143
+ clone_call = CloneCallType(
144
+ cell_to_clones_mapping=cell_to_clones_mapping,
145
+ clone_to_cells_mapping=clone_to_cells_mapping,
146
+ clone_intbc_sets=clone_intbc_sets,
147
+ cell_intbc_sets=cell_intbc_sets,
148
+ )
149
+ return clone_call
150
+
151
+
152
+ def test_call_lineage_groups():
153
+ """
154
+ Just tests that the clone calling algorithm runs on a trivial case.
155
+
156
+ In this test case, there are two clones characterized by the intBC sets
157
+ ["AAA", "BBB", "CCC"] and ["DDD", "EEE", "FFF"] respectively. The first
158
+ clone contains the cellBCs ["cellBC_1", "cellBC_2"] whereas the second
159
+ clone contains the cellBCs ["cellBC_3", "cellBC_4"].
160
+ """
161
+ intbc_table = pd.DataFrame(
162
+ data=[
163
+ ["cellBC_1", "AAA", 40],
164
+ ["cellBC_1", "BBB", 40],
165
+ ["cellBC_1", "CCC", 40],
166
+ ["cellBC_2", "AAA", 40],
167
+ ["cellBC_2", "BBB", 40],
168
+ ["cellBC_2", "CCC", 40],
169
+ ["cellBC_3", "DDD", 40],
170
+ ["cellBC_3", "EEE", 40],
171
+ ["cellBC_3", "FFF", 40],
172
+ ["cellBC_4", "DDD", 40],
173
+ ["cellBC_4", "EEE", 40],
174
+ ["cellBC_4", "FFF", 40],
175
+ ["cellBC_5", "DDD", 40],
176
+ ["cellBC_5", "EEE", 40],
177
+ ["cellBC_5", "FFF", 40],
178
+ ],
179
+ columns=["cellBC", "intBC", "UMI_count"],
180
+ )
181
+
182
+ clone_call = call_lineage_groups(
183
+ intbc_table_per_sample={"sample1": intbc_table},
184
+ min_umi_per_cell=1,
185
+ min_avg_reads_per_umi=2.0,
186
+ min_cluster_prop=0.005,
187
+ min_intbc_thresh=0.05,
188
+ inter_doublet_threshold=0.35,
189
+ kinship_thresh=0.25,
190
+ plot=False,
191
+ )
192
+
193
+ _check_are_clones(
194
+ clone_call=clone_call,
195
+ expected_clones=[
196
+ ["sample1_cellBC_1", "sample1_cellBC_2"],
197
+ ["sample1_cellBC_3", "sample1_cellBC_4", "sample1_cellBC_5"],
198
+ ],
199
+ )
200
+ assert clone_call.clone_intbc_sets["clone_2"] == set(["AAA", "BBB", "CCC"])
201
+ assert clone_call.clone_intbc_sets["clone_1"] == set(["DDD", "EEE", "FFF"])
202
+ assert clone_call.cell_to_clones_mapping["sample1_cellBC_1"] == set(["clone_2"])
203
+ assert clone_call.cell_to_clones_mapping["sample1_cellBC_2"] == set(["clone_2"])
204
+ assert clone_call.cell_to_clones_mapping["sample1_cellBC_3"] == set(["clone_1"])
205
+ assert clone_call.cell_to_clones_mapping["sample1_cellBC_4"] == set(["clone_1"])
206
+ assert clone_call.cell_to_clones_mapping["sample1_cellBC_5"] == set(["clone_1"])
207
+
208
+
209
+ def test_call_lineage_groups_doublet():
210
+ """
211
+ We check that cassiopeia can find the doublet.
212
+ """
213
+ ncells_per_clone = 20
214
+ intbc_table = pd.DataFrame(
215
+ data=[[f"cellBC_{i}", "AAA", 40] for i in range(ncells_per_clone)]
216
+ + [[f"cellBC_{i}", "BBB", 40] for i in range(ncells_per_clone)]
217
+ + [[f"cellBC_{i}", "CCC", 40] for i in range(ncells_per_clone)]
218
+ + [
219
+ [f"cellBC_{i}", "DDD", 40]
220
+ for i in range(
221
+ ncells_per_clone, 2 * ncells_per_clone + 1
222
+ ) # One more cell to ensure it is the first clone
223
+ ]
224
+ + [
225
+ [f"cellBC_{i}", "EEE", 40]
226
+ for i in range(
227
+ ncells_per_clone, 2 * ncells_per_clone + 1
228
+ ) # One more cell to ensure it is the first clone
229
+ ]
230
+ + [
231
+ [f"cellBC_{i}", "FFF", 40]
232
+ for i in range(
233
+ ncells_per_clone, 2 * ncells_per_clone + 1
234
+ ) # One more cell to ensure it is the first clone
235
+ ]
236
+ + [
237
+ ["cellBC_doublet", "AAA", 40],
238
+ ["cellBC_doublet", "BBB", 40],
239
+ ["cellBC_doublet", "CCC", 40],
240
+ ["cellBC_doublet", "DDD", 40],
241
+ ["cellBC_doublet", "EEE", 40],
242
+ ["cellBC_doublet", "FFF", 40],
243
+ ],
244
+ columns=["cellBC", "intBC", "UMI_count"],
245
+ )
246
+
247
+ clone_call = call_lineage_groups(
248
+ intbc_table_per_sample={"sample1": intbc_table},
249
+ min_umi_per_cell=1,
250
+ min_avg_reads_per_umi=2.0,
251
+ min_cluster_prop=0.005,
252
+ min_intbc_thresh=0.05,
253
+ inter_doublet_threshold=0.55, # With default value 0.35, the doublet is not detected! Indeed, it get's kinship 0.5 for both clones.
254
+ kinship_thresh=0.25,
255
+ plot=False,
256
+ )
257
+
258
+ _check_are_clones(
259
+ clone_call=clone_call,
260
+ expected_clones=[
261
+ [f"sample1_cellBC_{i}" for i in range(ncells_per_clone)],
262
+ [
263
+ f"sample1_cellBC_{i}"
264
+ for i in range(ncells_per_clone, 2 * ncells_per_clone + 1)
265
+ ],
266
+ ],
267
+ )
268
+ assert clone_call.clone_intbc_sets["clone_2"] == set(["AAA", "BBB", "CCC"])
269
+ assert clone_call.clone_intbc_sets["clone_1"] == set(["DDD", "EEE", "FFF"])
270
+ # assert(clone_call.cell_to_clones_mapping["sample1_cellBC_doublet"] == set(["clone_0", "clone_1"])) # Cassiopeia code does not return doublet identity.
@@ -0,0 +1,24 @@
1
+ """
2
+ Caching decorator module.
3
+
4
+ The caching decorator module provides decorators to wrap functions such that
5
+ any previously done computation is avoided in subsequent function calls. The
6
+ caching is performed directly in the filesystem (as opposed to in-memory),
7
+ enabling persistent caching. This is fundamental for compute-intensive
8
+ applications.
9
+ """
10
+
11
+ from ._cached import cached
12
+ from ._cached_computation import cached_computation
13
+ from ._cached_parallel_computation import (
14
+ cached_parallel_computation,
15
+ secure_parallel_output,
16
+ )
17
+ from ._common import (
18
+ set_cache_dir,
19
+ set_dir_levels,
20
+ set_hash_len,
21
+ set_log_level,
22
+ set_read_only,
23
+ set_use_hash,
24
+ )
@@ -0,0 +1,175 @@
1
+ import logging
2
+ import os
3
+ import pickle
4
+ from functools import wraps
5
+ from inspect import signature
6
+ from typing import List, Optional
7
+
8
+ from ._common import (CacheUsageError, _get_mode, _hash_all, get_cache_dir,
9
+ get_read_only, get_use_hash)
10
+
11
+ logger = logging.getLogger('.'.join(__name__.split('.')[:-1]))
12
+
13
+
14
+ def cached(
15
+ exclude: Optional[List] = None,
16
+ exclude_if_default: Optional[List] = None,
17
+ ):
18
+ """
19
+ Cache the wrapped function.
20
+ The outputs of the wrapped function are cached by pickling them into a
21
+ file whose path is determined by the function's name and it's arguments.
22
+ By default all arguments are used to define the cache key, but they can be
23
+ excluded via the `exclude` list. Moreover, an argument can be excluded
24
+ *only* if it has default values via the `exclude_if_default` list.
25
+ Args:
26
+ exclude: What arguments to exclude from the hash key. E.g.
27
+ n_processes, which does not affect the result of the function.
28
+ If None, nothing will be excluded.
29
+ exclude_if_default: What arguments to exclude from the hash key *if
30
+ default*. E.g. new arguments introduced to a function as it is
31
+ extended with new functionality, while preserving the old
32
+ behavior when it has default values.
33
+ """
34
+
35
+ def caching_decorator(func):
36
+ @wraps(func)
37
+ def wrapper(*args, **kwargs):
38
+ cache_dir = get_cache_dir()
39
+ if cache_dir is None:
40
+ return func(*args, **kwargs)
41
+ use_hash = get_use_hash()
42
+
43
+ s = signature(func)
44
+ # Check that all excluded args are present -
45
+ # user might have a typo!
46
+ if exclude is not None:
47
+ for arg in exclude:
48
+ if arg not in s.parameters:
49
+ raise CacheUsageError(
50
+ f"{arg} is not an argument to {func.__name__}. "
51
+ "Fix the arguments in `exclude`."
52
+ )
53
+ # Check that all exclude_if_default args are present -
54
+ # user might have a typo!
55
+ if exclude_if_default is not None:
56
+ for arg in exclude_if_default:
57
+ if arg not in s.parameters:
58
+ raise CacheUsageError(
59
+ f"{arg} is not an argument to {func.__name__}. "
60
+ "Fix the arguments in `exclude_if_default`."
61
+ )
62
+
63
+ # None of the exclude_if_default args can be a prefix of another one
64
+ # (otherwise adversarial hash collisions can be easily crafted due
65
+ # to how args are concatenated and hashed)
66
+ if exclude_if_default is not None:
67
+ for arg1 in exclude_if_default:
68
+ for arg2 in exclude_if_default:
69
+ if arg1 != arg2 and arg2.startswith(arg1):
70
+ raise CacheUsageError(
71
+ "None of the exclude_if_default arguments can "
72
+ "be a prefix of another exclude_if_default "
73
+ "argument. This ensures that it is not "
74
+ "possible to craft adversarial hash collisions."
75
+ )
76
+
77
+ def excluded(arg, val) -> bool:
78
+ if exclude is not None and arg in exclude:
79
+ return True
80
+ if exclude_if_default is not None and arg in exclude_if_default:
81
+ p = s.parameters[arg]
82
+ default = p.default
83
+ if val == default:
84
+ return True
85
+ return False
86
+
87
+ binding = s.bind(*args, **kwargs)
88
+ binding.apply_defaults()
89
+ if not use_hash:
90
+ path = (
91
+ [cache_dir]
92
+ + [f"{func.__name__}"]
93
+ + [
94
+ f"{arg}_{val}"
95
+ for (arg, val) in binding.arguments.items()
96
+ if (not excluded(arg, val))
97
+ ]
98
+ + ["result"]
99
+ )
100
+ else:
101
+ path = (
102
+ [cache_dir]
103
+ + [f"{func.__name__}"]
104
+ + [
105
+ _hash_all(
106
+ sum(
107
+ [
108
+ [f"{arg}", f"{val}"]
109
+ for (arg, val) in binding.arguments.items()
110
+ if not excluded(arg, val)
111
+ ],
112
+ [],
113
+ )
114
+ )
115
+ ]
116
+ + ["result"]
117
+ )
118
+ success_token_filename = os.path.join(*path) + ".success"
119
+ filename = os.path.join(*path) + ".pickle"
120
+
121
+ def computed():
122
+ # Check that each of the output files exists
123
+ if not os.path.exists(filename):
124
+ return False
125
+ # Not checking mode for backwards compatibility reasons.
126
+ # mode = _get_mode(filename)
127
+ # if mode != "444":
128
+ # return False
129
+
130
+ if not os.path.exists(success_token_filename):
131
+ return False
132
+ return True
133
+
134
+ def clear_previous_outputs():
135
+ if os.path.exists(filename):
136
+ logger.info(f"Removing possibly corrupted {filename}")
137
+ os.system(f'chmod 666 "{filename}"')
138
+ os.remove(filename)
139
+
140
+ if os.path.exists(success_token_filename):
141
+ logger.info(f"Removing {success_token_filename}")
142
+ os.system(f'chmod 666 "{success_token_filename}"')
143
+ os.remove(success_token_filename)
144
+
145
+ # Only call the function if there is any work to do at all.
146
+ if not computed():
147
+ # Now call the wrapped function
148
+ logger.debug(
149
+ f"Calling {func.__name__} . Output location: {filename}"
150
+ )
151
+ if get_read_only():
152
+ raise CacheUsageError(
153
+ "Cache is in read only mode! Will not call function."
154
+ )
155
+ clear_previous_outputs()
156
+ res = func(*args, **kwargs)
157
+
158
+ os.umask(0) # To ensure all collaborators can access cache
159
+ os.makedirs(os.path.join(*path[:-1]), exist_ok=True, mode=0o777)
160
+ with open(filename, "wb") as f:
161
+ pickle.dump(res, f)
162
+ f.flush()
163
+ os.system(f'chmod 444 "{filename}"')
164
+
165
+ with open(success_token_filename, "w") as f:
166
+ f.write("SUCCESS\n")
167
+ f.flush()
168
+ return res
169
+ else:
170
+ with open(filename, "rb") as f:
171
+ return pickle.load(f)
172
+
173
+ return wrapper
174
+
175
+ return caching_decorator