nova-clone 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nova_clone/__init__.py +1 -0
- nova_clone/algorithm/__init__.py +2 -0
- nova_clone/algorithm/_build_intbc_graph.py +135 -0
- nova_clone/algorithm/_call_clones_from_coresets.py +235 -0
- nova_clone/algorithm/_clone_call_ops.py +258 -0
- nova_clone/algorithm/_clone_calling_algorithm.py +1571 -0
- nova_clone/algorithm/_compute_cellbc_vs_coreset_overlap_matrix.py +143 -0
- nova_clone/algorithm/_compute_coreset_co_assignment_matrix.py +170 -0
- nova_clone/algorithm/_find_maximal_coresets.py +428 -0
- nova_clone/algorithm/_keep_reliable_edges.py +202 -0
- nova_clone/algorithm/_make_coresets_disjoint.py +114 -0
- nova_clone/algorithm/_prune_coresets.py +164 -0
- nova_clone/algorithm/_prune_custom_intbcs.py +150 -0
- nova_clone/algorithm/_prune_likely_homoplastic_intbcs.py +162 -0
- nova_clone/algorithm/_prune_likely_sequencing_errors.py +201 -0
- nova_clone/algorithm/_prune_low_cellbc_count_intbcs.py +154 -0
- nova_clone/algorithm/_prune_low_umi_count_intbcs.py +152 -0
- nova_clone/algorithm/_types.py +150 -0
- nova_clone/algorithm/_utils.py +53 -0
- nova_clone/algorithm/cassiopeia/__init__.py +1 -0
- nova_clone/algorithm/cassiopeia/_call_lineage_groups.py +270 -0
- nova_clone/caching/__init__.py +24 -0
- nova_clone/caching/_cached.py +175 -0
- nova_clone/caching/_cached_computation.py +369 -0
- nova_clone/caching/_cached_parallel_computation.py +428 -0
- nova_clone/caching/_common.py +200 -0
- nova_clone/data/__init__.py +1 -0
- nova_clone/qc/__init__.py +179 -0
- nova_clone/qc/_clone_intbc_sets.py +234 -0
- nova_clone/qc/_compare_clone_calls.py +218 -0
- nova_clone/qc/_data_retention.py +175 -0
- nova_clone/qc/_homoplasy.py +81 -0
- nova_clone/qc/_inter_clonal_kinship.py +60 -0
- nova_clone/qc/_intra_clonal_kinship.py +172 -0
- nova_clone/qc/_multiplet_rate.py +158 -0
- nova_clone/qc/_number_of_clones.py +153 -0
- nova_clone/qc/_pre_qc.py +485 -0
- nova_clone/simulation/HDBSCAN_method_adjusted.py +68 -0
- nova_clone/simulation/__init__.py +0 -0
- nova_clone/simulation/_caching.py +40 -0
- nova_clone/simulation/benchmarking.py +1221 -0
- nova_clone/simulation/clone_to_sequences.py +408 -0
- nova_clone/simulation/creating_tree.py +416 -0
- nova_clone/simulation/evaluation.py +344 -0
- nova_clone/simulation/jaccard_hierarchical_by_intBC.py +78 -0
- nova_clone/simulation/plot_benchmark_lines.py +226 -0
- nova_clone/simulation/slice_benchmark.py +640 -0
- nova_clone/simulation/slice_benchmark_clone_calling.py +68 -0
- nova_clone-0.1.0.dist-info/METADATA +298 -0
- nova_clone-0.1.0.dist-info/RECORD +53 -0
- nova_clone-0.1.0.dist-info/WHEEL +5 -0
- nova_clone-0.1.0.dist-info/licenses/LICENSE +28 -0
- nova_clone-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
from typing import Set
|
|
2
|
+
|
|
3
|
+
from ._types import *
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def _cell_intbc_sets_with_counts__for_intbc_table(
|
|
7
|
+
intbc_table: IntBCTableType,
|
|
8
|
+
) -> Dict[CellBCType, Dict[IntBCType, UMICountType]]:
|
|
9
|
+
"""
|
|
10
|
+
Get the intBC sets corresponding to each cell, w/#UMIs.
|
|
11
|
+
"""
|
|
12
|
+
validate_intbc_table(intbc_table)
|
|
13
|
+
cell_intbc_sets_with_counts = {}
|
|
14
|
+
for cellbc, cell_df in intbc_table.groupby(
|
|
15
|
+
"cellBC"
|
|
16
|
+
): # Not using tqdm bc its fast (<10 sec)
|
|
17
|
+
# Lets see what intBCs this cell has
|
|
18
|
+
intbc_and_umi_count = list(zip(list(cell_df.intBC), list(cell_df.UMI_count)))
|
|
19
|
+
cell_intbc_sets_with_counts[cellbc] = dict(intbc_and_umi_count)
|
|
20
|
+
return cell_intbc_sets_with_counts
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _cell_intbc_set_from_intbc_table(
|
|
24
|
+
intbc_table: IntBCTableType,
|
|
25
|
+
) -> Dict[str, Set[str]]:
|
|
26
|
+
"""
|
|
27
|
+
Get the intBC sets corresponding to each cell.
|
|
28
|
+
"""
|
|
29
|
+
cell_intbc_sets_with_counts = _cell_intbc_sets_with_counts__for_intbc_table(
|
|
30
|
+
intbc_table=intbc_table,
|
|
31
|
+
)
|
|
32
|
+
cell_intbc_sets = {
|
|
33
|
+
cell: {intbc for (intbc, counts) in intbcs_with_counts.items()}
|
|
34
|
+
for (cell, intbcs_with_counts) in cell_intbc_sets_with_counts.items()
|
|
35
|
+
}
|
|
36
|
+
return cell_intbc_sets
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _cell_intbc_sets_from_intbc_tables(
|
|
40
|
+
intbc_table_per_sample: Dict[SampleNameType, IntBCTableType],
|
|
41
|
+
) -> Dict[str, Set[str]]:
|
|
42
|
+
"""
|
|
43
|
+
Get the intBC sets corresponding to each cell.
|
|
44
|
+
"""
|
|
45
|
+
res = {}
|
|
46
|
+
for sample_name, intbc_table in intbc_table_per_sample.items():
|
|
47
|
+
cell_intbc_sets = _cell_intbc_set_from_intbc_table(
|
|
48
|
+
intbc_table=intbc_table,
|
|
49
|
+
)
|
|
50
|
+
for x, y in cell_intbc_sets.items():
|
|
51
|
+
assert x not in res
|
|
52
|
+
res[f"{sample_name}_" + x] = y
|
|
53
|
+
return res
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from ._call_lineage_groups import call_lineage_groups
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
from copy import deepcopy
|
|
2
|
+
|
|
3
|
+
from cassiopeia.preprocess.pipeline import (
|
|
4
|
+
call_lineage_groups as call_lineage_groups_cassiopeia,
|
|
5
|
+
)
|
|
6
|
+
from nova_clone.algorithm._prune_low_umi_count_intbcs import prune_low_umi_count_intbcs
|
|
7
|
+
|
|
8
|
+
from nova_clone.algorithm._prune_low_umi_count_intbcs import (
|
|
9
|
+
prune_low_umi_count_intbcs,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
from .._clone_calling_algorithm import _check_are_clones
|
|
13
|
+
from .._types import *
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _build_input_df(intbc_table_per_sample) -> pd.DataFrame:
|
|
17
|
+
rows = []
|
|
18
|
+
for sample_name, intbc_table in intbc_table_per_sample.items():
|
|
19
|
+
for _, row in intbc_table.iterrows():
|
|
20
|
+
row_dict = row.to_dict()
|
|
21
|
+
res = {
|
|
22
|
+
"cellBC": f"{sample_name}_{row_dict['cellBC']}",
|
|
23
|
+
"UMI": row_dict["UMI_count"],
|
|
24
|
+
"intBC": row_dict["intBC"],
|
|
25
|
+
"allele": "A",
|
|
26
|
+
"readCount": 1000,
|
|
27
|
+
}
|
|
28
|
+
rows.append(res)
|
|
29
|
+
return pd.DataFrame(rows)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def call_lineage_groups(
|
|
33
|
+
intbc_table_per_sample: Dict[SampleNameType, IntBCTableType],
|
|
34
|
+
min_umi_per_cell: int = 10,
|
|
35
|
+
min_avg_reads_per_umi: float = 2.0,
|
|
36
|
+
min_cluster_prop: float = 0.005,
|
|
37
|
+
min_intbc_thresh: float = 0.05,
|
|
38
|
+
inter_doublet_threshold: float = 0.35,
|
|
39
|
+
kinship_thresh: float = 0.25,
|
|
40
|
+
plot: bool = False,
|
|
41
|
+
min_intbc_umi_count: int = 0, # Default value is 0 because it is not used in the cassiopeia code. This is expected to be part of user preprocessing.
|
|
42
|
+
) -> CloneCallType:
|
|
43
|
+
"""
|
|
44
|
+
Wrapper around Cassiopeia's call_lineage_groups function.
|
|
45
|
+
|
|
46
|
+
Assigns cells to their clonal populations.
|
|
47
|
+
|
|
48
|
+
Performs multiple rounds of filtering and assigning to lineage groups:
|
|
49
|
+
1. Iteratively generates putative lineage groups by forming intBC
|
|
50
|
+
groups for each lineage group and then assigning cells based on how
|
|
51
|
+
many intBCs they share with each intBC group (kinship).
|
|
52
|
+
|
|
53
|
+
2. Refines these putative groups by removing non-informative intBCs
|
|
54
|
+
and reassigning cells through kinship.
|
|
55
|
+
|
|
56
|
+
3. Removes all inter-lineage doublets, defined as cells that have
|
|
57
|
+
relatively equal kinship scores across multiple lineages and whose
|
|
58
|
+
assignments are therefore ambigious.
|
|
59
|
+
|
|
60
|
+
4. Finally, performs one more round of filtering non-informative intBCs
|
|
61
|
+
and cellBCs with low UMI counts before returning a final table of
|
|
62
|
+
lineage assignments, allele information, and read and umi counts for
|
|
63
|
+
each sample.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
intbc_table_per_sample: The intBC table per sample.
|
|
67
|
+
min_umi_per_cell: The threshold specifying the minimum number of UMIs a
|
|
68
|
+
cell needs in order to not be filtered during filtering
|
|
69
|
+
min_avg_reads_per_umi: The threshold specifying the minimum coverage
|
|
70
|
+
(i.e. average) reads per UMI in a cell needed in order for that
|
|
71
|
+
cell not to be filtered during filtering
|
|
72
|
+
min_cluster_prop: The minimum cluster size in the putative lineage
|
|
73
|
+
assignment step, as a proportion of the number of cells
|
|
74
|
+
min_intbc_thresh: The threshold specifying the minimum proportion of
|
|
75
|
+
cells in a lineage group that need to have an intBC in order for it
|
|
76
|
+
be retained during filtering. Also specifies the minimum proportion
|
|
77
|
+
of cells that share an intBC with the most frequent intBC in
|
|
78
|
+
forming putative lineage groups
|
|
79
|
+
inter_doublet_threshold: The threshold specifying the minimum proportion
|
|
80
|
+
of kinship a cell shares with its assigned lineage group out of all
|
|
81
|
+
lineage groups for it to be retained during doublet filtering
|
|
82
|
+
kinship_thresh: The threshold specifying the minimum proportion of
|
|
83
|
+
intBCs shared between a cell and the intBC set of a lineage group
|
|
84
|
+
needed to assign that cell to that lineage group in putative
|
|
85
|
+
assignment
|
|
86
|
+
plot: Indicates whether to generate plots. If so, they will be written to
|
|
87
|
+
`output_directory_cassiopeia`.
|
|
88
|
+
min_intbc_umi_count: The minimum number of UMIs for an intBC to be considered
|
|
89
|
+
in a cellBC.
|
|
90
|
+
|
|
91
|
+
Returns:
|
|
92
|
+
The clone call, with fields `clone_to_cells_mapping, cell_to_clones_mapping, clone_intbc_sets, cell_intbc_sets`.
|
|
93
|
+
"""
|
|
94
|
+
validate_intbc_table_per_sample(intbc_table_per_sample=intbc_table_per_sample)
|
|
95
|
+
original_intbc_table_per_sample = deepcopy(intbc_table_per_sample)
|
|
96
|
+
intbc_table_per_sample = deepcopy(intbc_table_per_sample)
|
|
97
|
+
|
|
98
|
+
# Filter intBCs based on min_intbc_umi_count
|
|
99
|
+
intbc_table_per_sample = prune_low_umi_count_intbcs(
|
|
100
|
+
intbc_table_per_sample=intbc_table_per_sample,
|
|
101
|
+
min_intbc_umi_count=min_intbc_umi_count,
|
|
102
|
+
).intbc_table_per_sample
|
|
103
|
+
|
|
104
|
+
input_df = _build_input_df(intbc_table_per_sample)
|
|
105
|
+
lineage_groups = call_lineage_groups_cassiopeia(
|
|
106
|
+
input_df=input_df,
|
|
107
|
+
output_directory="output_directory_cassiopeia",
|
|
108
|
+
min_umi_per_cell=min_umi_per_cell,
|
|
109
|
+
min_avg_reads_per_umi=min_avg_reads_per_umi,
|
|
110
|
+
min_cluster_prop=min_cluster_prop,
|
|
111
|
+
min_intbc_thresh=min_intbc_thresh,
|
|
112
|
+
inter_doublet_threshold=inter_doublet_threshold,
|
|
113
|
+
kinship_thresh=kinship_thresh,
|
|
114
|
+
plot=plot,
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
cell_to_clones_mapping = {}
|
|
118
|
+
cell_intbc_sets = {}
|
|
119
|
+
clone_intbc_sets = defaultdict(set)
|
|
120
|
+
for _, row in lineage_groups.iterrows():
|
|
121
|
+
row = row.to_dict()
|
|
122
|
+
clone = "clone_" + str(row["lineageGrp"])
|
|
123
|
+
cell = str(row["cellBC"])
|
|
124
|
+
if cell not in cell_to_clones_mapping:
|
|
125
|
+
cell_to_clones_mapping[cell] = set()
|
|
126
|
+
cell_to_clones_mapping[cell].add(clone)
|
|
127
|
+
clone_intbc_sets[clone].add(row["intBC"])
|
|
128
|
+
if cell not in cell_intbc_sets:
|
|
129
|
+
cell_intbc_sets[cell] = set()
|
|
130
|
+
cell_intbc_sets[cell].add(row["intBC"])
|
|
131
|
+
# Add cellBCs that were not assigned to any clone
|
|
132
|
+
for sample, intbc_table in original_intbc_table_per_sample.items():
|
|
133
|
+
for _, row in intbc_table.iterrows():
|
|
134
|
+
row = row.to_dict()
|
|
135
|
+
cellbc = f"{sample}_{row['cellBC']}"
|
|
136
|
+
if cellbc not in cell_to_clones_mapping:
|
|
137
|
+
cell_to_clones_mapping[cellbc] = set()
|
|
138
|
+
clone_to_cells_mapping = defaultdict(set)
|
|
139
|
+
for cell, clones in cell_to_clones_mapping.items():
|
|
140
|
+
for clone in clones:
|
|
141
|
+
clone_to_cells_mapping[clone].add(cell)
|
|
142
|
+
clone_intbc_sets = {x: y for (x, y) in clone_intbc_sets.items()}
|
|
143
|
+
clone_call = CloneCallType(
|
|
144
|
+
cell_to_clones_mapping=cell_to_clones_mapping,
|
|
145
|
+
clone_to_cells_mapping=clone_to_cells_mapping,
|
|
146
|
+
clone_intbc_sets=clone_intbc_sets,
|
|
147
|
+
cell_intbc_sets=cell_intbc_sets,
|
|
148
|
+
)
|
|
149
|
+
return clone_call
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def test_call_lineage_groups():
|
|
153
|
+
"""
|
|
154
|
+
Just tests that the clone calling algorithm runs on a trivial case.
|
|
155
|
+
|
|
156
|
+
In this test case, there are two clones characterized by the intBC sets
|
|
157
|
+
["AAA", "BBB", "CCC"] and ["DDD", "EEE", "FFF"] respectively. The first
|
|
158
|
+
clone contains the cellBCs ["cellBC_1", "cellBC_2"] whereas the second
|
|
159
|
+
clone contains the cellBCs ["cellBC_3", "cellBC_4"].
|
|
160
|
+
"""
|
|
161
|
+
intbc_table = pd.DataFrame(
|
|
162
|
+
data=[
|
|
163
|
+
["cellBC_1", "AAA", 40],
|
|
164
|
+
["cellBC_1", "BBB", 40],
|
|
165
|
+
["cellBC_1", "CCC", 40],
|
|
166
|
+
["cellBC_2", "AAA", 40],
|
|
167
|
+
["cellBC_2", "BBB", 40],
|
|
168
|
+
["cellBC_2", "CCC", 40],
|
|
169
|
+
["cellBC_3", "DDD", 40],
|
|
170
|
+
["cellBC_3", "EEE", 40],
|
|
171
|
+
["cellBC_3", "FFF", 40],
|
|
172
|
+
["cellBC_4", "DDD", 40],
|
|
173
|
+
["cellBC_4", "EEE", 40],
|
|
174
|
+
["cellBC_4", "FFF", 40],
|
|
175
|
+
["cellBC_5", "DDD", 40],
|
|
176
|
+
["cellBC_5", "EEE", 40],
|
|
177
|
+
["cellBC_5", "FFF", 40],
|
|
178
|
+
],
|
|
179
|
+
columns=["cellBC", "intBC", "UMI_count"],
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
clone_call = call_lineage_groups(
|
|
183
|
+
intbc_table_per_sample={"sample1": intbc_table},
|
|
184
|
+
min_umi_per_cell=1,
|
|
185
|
+
min_avg_reads_per_umi=2.0,
|
|
186
|
+
min_cluster_prop=0.005,
|
|
187
|
+
min_intbc_thresh=0.05,
|
|
188
|
+
inter_doublet_threshold=0.35,
|
|
189
|
+
kinship_thresh=0.25,
|
|
190
|
+
plot=False,
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
_check_are_clones(
|
|
194
|
+
clone_call=clone_call,
|
|
195
|
+
expected_clones=[
|
|
196
|
+
["sample1_cellBC_1", "sample1_cellBC_2"],
|
|
197
|
+
["sample1_cellBC_3", "sample1_cellBC_4", "sample1_cellBC_5"],
|
|
198
|
+
],
|
|
199
|
+
)
|
|
200
|
+
assert clone_call.clone_intbc_sets["clone_2"] == set(["AAA", "BBB", "CCC"])
|
|
201
|
+
assert clone_call.clone_intbc_sets["clone_1"] == set(["DDD", "EEE", "FFF"])
|
|
202
|
+
assert clone_call.cell_to_clones_mapping["sample1_cellBC_1"] == set(["clone_2"])
|
|
203
|
+
assert clone_call.cell_to_clones_mapping["sample1_cellBC_2"] == set(["clone_2"])
|
|
204
|
+
assert clone_call.cell_to_clones_mapping["sample1_cellBC_3"] == set(["clone_1"])
|
|
205
|
+
assert clone_call.cell_to_clones_mapping["sample1_cellBC_4"] == set(["clone_1"])
|
|
206
|
+
assert clone_call.cell_to_clones_mapping["sample1_cellBC_5"] == set(["clone_1"])
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def test_call_lineage_groups_doublet():
|
|
210
|
+
"""
|
|
211
|
+
We check that cassiopeia can find the doublet.
|
|
212
|
+
"""
|
|
213
|
+
ncells_per_clone = 20
|
|
214
|
+
intbc_table = pd.DataFrame(
|
|
215
|
+
data=[[f"cellBC_{i}", "AAA", 40] for i in range(ncells_per_clone)]
|
|
216
|
+
+ [[f"cellBC_{i}", "BBB", 40] for i in range(ncells_per_clone)]
|
|
217
|
+
+ [[f"cellBC_{i}", "CCC", 40] for i in range(ncells_per_clone)]
|
|
218
|
+
+ [
|
|
219
|
+
[f"cellBC_{i}", "DDD", 40]
|
|
220
|
+
for i in range(
|
|
221
|
+
ncells_per_clone, 2 * ncells_per_clone + 1
|
|
222
|
+
) # One more cell to ensure it is the first clone
|
|
223
|
+
]
|
|
224
|
+
+ [
|
|
225
|
+
[f"cellBC_{i}", "EEE", 40]
|
|
226
|
+
for i in range(
|
|
227
|
+
ncells_per_clone, 2 * ncells_per_clone + 1
|
|
228
|
+
) # One more cell to ensure it is the first clone
|
|
229
|
+
]
|
|
230
|
+
+ [
|
|
231
|
+
[f"cellBC_{i}", "FFF", 40]
|
|
232
|
+
for i in range(
|
|
233
|
+
ncells_per_clone, 2 * ncells_per_clone + 1
|
|
234
|
+
) # One more cell to ensure it is the first clone
|
|
235
|
+
]
|
|
236
|
+
+ [
|
|
237
|
+
["cellBC_doublet", "AAA", 40],
|
|
238
|
+
["cellBC_doublet", "BBB", 40],
|
|
239
|
+
["cellBC_doublet", "CCC", 40],
|
|
240
|
+
["cellBC_doublet", "DDD", 40],
|
|
241
|
+
["cellBC_doublet", "EEE", 40],
|
|
242
|
+
["cellBC_doublet", "FFF", 40],
|
|
243
|
+
],
|
|
244
|
+
columns=["cellBC", "intBC", "UMI_count"],
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
clone_call = call_lineage_groups(
|
|
248
|
+
intbc_table_per_sample={"sample1": intbc_table},
|
|
249
|
+
min_umi_per_cell=1,
|
|
250
|
+
min_avg_reads_per_umi=2.0,
|
|
251
|
+
min_cluster_prop=0.005,
|
|
252
|
+
min_intbc_thresh=0.05,
|
|
253
|
+
inter_doublet_threshold=0.55, # With default value 0.35, the doublet is not detected! Indeed, it get's kinship 0.5 for both clones.
|
|
254
|
+
kinship_thresh=0.25,
|
|
255
|
+
plot=False,
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
_check_are_clones(
|
|
259
|
+
clone_call=clone_call,
|
|
260
|
+
expected_clones=[
|
|
261
|
+
[f"sample1_cellBC_{i}" for i in range(ncells_per_clone)],
|
|
262
|
+
[
|
|
263
|
+
f"sample1_cellBC_{i}"
|
|
264
|
+
for i in range(ncells_per_clone, 2 * ncells_per_clone + 1)
|
|
265
|
+
],
|
|
266
|
+
],
|
|
267
|
+
)
|
|
268
|
+
assert clone_call.clone_intbc_sets["clone_2"] == set(["AAA", "BBB", "CCC"])
|
|
269
|
+
assert clone_call.clone_intbc_sets["clone_1"] == set(["DDD", "EEE", "FFF"])
|
|
270
|
+
# assert(clone_call.cell_to_clones_mapping["sample1_cellBC_doublet"] == set(["clone_0", "clone_1"])) # Cassiopeia code does not return doublet identity.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Caching decorator module.
|
|
3
|
+
|
|
4
|
+
The caching decorator module provides decorators to wrap functions such that
|
|
5
|
+
any previously done computation is avoided in subsequent function calls. The
|
|
6
|
+
caching is performed directly in the filesystem (as opposed to in-memory),
|
|
7
|
+
enabling persistent caching. This is fundamental for compute-intensive
|
|
8
|
+
applications.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from ._cached import cached
|
|
12
|
+
from ._cached_computation import cached_computation
|
|
13
|
+
from ._cached_parallel_computation import (
|
|
14
|
+
cached_parallel_computation,
|
|
15
|
+
secure_parallel_output,
|
|
16
|
+
)
|
|
17
|
+
from ._common import (
|
|
18
|
+
set_cache_dir,
|
|
19
|
+
set_dir_levels,
|
|
20
|
+
set_hash_len,
|
|
21
|
+
set_log_level,
|
|
22
|
+
set_read_only,
|
|
23
|
+
set_use_hash,
|
|
24
|
+
)
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
import pickle
|
|
4
|
+
from functools import wraps
|
|
5
|
+
from inspect import signature
|
|
6
|
+
from typing import List, Optional
|
|
7
|
+
|
|
8
|
+
from ._common import (CacheUsageError, _get_mode, _hash_all, get_cache_dir,
|
|
9
|
+
get_read_only, get_use_hash)
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger('.'.join(__name__.split('.')[:-1]))
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def cached(
|
|
15
|
+
exclude: Optional[List] = None,
|
|
16
|
+
exclude_if_default: Optional[List] = None,
|
|
17
|
+
):
|
|
18
|
+
"""
|
|
19
|
+
Cache the wrapped function.
|
|
20
|
+
The outputs of the wrapped function are cached by pickling them into a
|
|
21
|
+
file whose path is determined by the function's name and it's arguments.
|
|
22
|
+
By default all arguments are used to define the cache key, but they can be
|
|
23
|
+
excluded via the `exclude` list. Moreover, an argument can be excluded
|
|
24
|
+
*only* if it has default values via the `exclude_if_default` list.
|
|
25
|
+
Args:
|
|
26
|
+
exclude: What arguments to exclude from the hash key. E.g.
|
|
27
|
+
n_processes, which does not affect the result of the function.
|
|
28
|
+
If None, nothing will be excluded.
|
|
29
|
+
exclude_if_default: What arguments to exclude from the hash key *if
|
|
30
|
+
default*. E.g. new arguments introduced to a function as it is
|
|
31
|
+
extended with new functionality, while preserving the old
|
|
32
|
+
behavior when it has default values.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def caching_decorator(func):
|
|
36
|
+
@wraps(func)
|
|
37
|
+
def wrapper(*args, **kwargs):
|
|
38
|
+
cache_dir = get_cache_dir()
|
|
39
|
+
if cache_dir is None:
|
|
40
|
+
return func(*args, **kwargs)
|
|
41
|
+
use_hash = get_use_hash()
|
|
42
|
+
|
|
43
|
+
s = signature(func)
|
|
44
|
+
# Check that all excluded args are present -
|
|
45
|
+
# user might have a typo!
|
|
46
|
+
if exclude is not None:
|
|
47
|
+
for arg in exclude:
|
|
48
|
+
if arg not in s.parameters:
|
|
49
|
+
raise CacheUsageError(
|
|
50
|
+
f"{arg} is not an argument to {func.__name__}. "
|
|
51
|
+
"Fix the arguments in `exclude`."
|
|
52
|
+
)
|
|
53
|
+
# Check that all exclude_if_default args are present -
|
|
54
|
+
# user might have a typo!
|
|
55
|
+
if exclude_if_default is not None:
|
|
56
|
+
for arg in exclude_if_default:
|
|
57
|
+
if arg not in s.parameters:
|
|
58
|
+
raise CacheUsageError(
|
|
59
|
+
f"{arg} is not an argument to {func.__name__}. "
|
|
60
|
+
"Fix the arguments in `exclude_if_default`."
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# None of the exclude_if_default args can be a prefix of another one
|
|
64
|
+
# (otherwise adversarial hash collisions can be easily crafted due
|
|
65
|
+
# to how args are concatenated and hashed)
|
|
66
|
+
if exclude_if_default is not None:
|
|
67
|
+
for arg1 in exclude_if_default:
|
|
68
|
+
for arg2 in exclude_if_default:
|
|
69
|
+
if arg1 != arg2 and arg2.startswith(arg1):
|
|
70
|
+
raise CacheUsageError(
|
|
71
|
+
"None of the exclude_if_default arguments can "
|
|
72
|
+
"be a prefix of another exclude_if_default "
|
|
73
|
+
"argument. This ensures that it is not "
|
|
74
|
+
"possible to craft adversarial hash collisions."
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
def excluded(arg, val) -> bool:
|
|
78
|
+
if exclude is not None and arg in exclude:
|
|
79
|
+
return True
|
|
80
|
+
if exclude_if_default is not None and arg in exclude_if_default:
|
|
81
|
+
p = s.parameters[arg]
|
|
82
|
+
default = p.default
|
|
83
|
+
if val == default:
|
|
84
|
+
return True
|
|
85
|
+
return False
|
|
86
|
+
|
|
87
|
+
binding = s.bind(*args, **kwargs)
|
|
88
|
+
binding.apply_defaults()
|
|
89
|
+
if not use_hash:
|
|
90
|
+
path = (
|
|
91
|
+
[cache_dir]
|
|
92
|
+
+ [f"{func.__name__}"]
|
|
93
|
+
+ [
|
|
94
|
+
f"{arg}_{val}"
|
|
95
|
+
for (arg, val) in binding.arguments.items()
|
|
96
|
+
if (not excluded(arg, val))
|
|
97
|
+
]
|
|
98
|
+
+ ["result"]
|
|
99
|
+
)
|
|
100
|
+
else:
|
|
101
|
+
path = (
|
|
102
|
+
[cache_dir]
|
|
103
|
+
+ [f"{func.__name__}"]
|
|
104
|
+
+ [
|
|
105
|
+
_hash_all(
|
|
106
|
+
sum(
|
|
107
|
+
[
|
|
108
|
+
[f"{arg}", f"{val}"]
|
|
109
|
+
for (arg, val) in binding.arguments.items()
|
|
110
|
+
if not excluded(arg, val)
|
|
111
|
+
],
|
|
112
|
+
[],
|
|
113
|
+
)
|
|
114
|
+
)
|
|
115
|
+
]
|
|
116
|
+
+ ["result"]
|
|
117
|
+
)
|
|
118
|
+
success_token_filename = os.path.join(*path) + ".success"
|
|
119
|
+
filename = os.path.join(*path) + ".pickle"
|
|
120
|
+
|
|
121
|
+
def computed():
|
|
122
|
+
# Check that each of the output files exists
|
|
123
|
+
if not os.path.exists(filename):
|
|
124
|
+
return False
|
|
125
|
+
# Not checking mode for backwards compatibility reasons.
|
|
126
|
+
# mode = _get_mode(filename)
|
|
127
|
+
# if mode != "444":
|
|
128
|
+
# return False
|
|
129
|
+
|
|
130
|
+
if not os.path.exists(success_token_filename):
|
|
131
|
+
return False
|
|
132
|
+
return True
|
|
133
|
+
|
|
134
|
+
def clear_previous_outputs():
|
|
135
|
+
if os.path.exists(filename):
|
|
136
|
+
logger.info(f"Removing possibly corrupted {filename}")
|
|
137
|
+
os.system(f'chmod 666 "{filename}"')
|
|
138
|
+
os.remove(filename)
|
|
139
|
+
|
|
140
|
+
if os.path.exists(success_token_filename):
|
|
141
|
+
logger.info(f"Removing {success_token_filename}")
|
|
142
|
+
os.system(f'chmod 666 "{success_token_filename}"')
|
|
143
|
+
os.remove(success_token_filename)
|
|
144
|
+
|
|
145
|
+
# Only call the function if there is any work to do at all.
|
|
146
|
+
if not computed():
|
|
147
|
+
# Now call the wrapped function
|
|
148
|
+
logger.debug(
|
|
149
|
+
f"Calling {func.__name__} . Output location: {filename}"
|
|
150
|
+
)
|
|
151
|
+
if get_read_only():
|
|
152
|
+
raise CacheUsageError(
|
|
153
|
+
"Cache is in read only mode! Will not call function."
|
|
154
|
+
)
|
|
155
|
+
clear_previous_outputs()
|
|
156
|
+
res = func(*args, **kwargs)
|
|
157
|
+
|
|
158
|
+
os.umask(0) # To ensure all collaborators can access cache
|
|
159
|
+
os.makedirs(os.path.join(*path[:-1]), exist_ok=True, mode=0o777)
|
|
160
|
+
with open(filename, "wb") as f:
|
|
161
|
+
pickle.dump(res, f)
|
|
162
|
+
f.flush()
|
|
163
|
+
os.system(f'chmod 444 "{filename}"')
|
|
164
|
+
|
|
165
|
+
with open(success_token_filename, "w") as f:
|
|
166
|
+
f.write("SUCCESS\n")
|
|
167
|
+
f.flush()
|
|
168
|
+
return res
|
|
169
|
+
else:
|
|
170
|
+
with open(filename, "rb") as f:
|
|
171
|
+
return pickle.load(f)
|
|
172
|
+
|
|
173
|
+
return wrapper
|
|
174
|
+
|
|
175
|
+
return caching_decorator
|