Oncodrive3D 1.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oncodrive3d-1.0.4.dist-info/METADATA +333 -0
- oncodrive3d-1.0.4.dist-info/RECORD +36 -0
- oncodrive3d-1.0.4.dist-info/WHEEL +4 -0
- oncodrive3d-1.0.4.dist-info/entry_points.txt +5 -0
- oncodrive3d-1.0.4.dist-info/licenses/LICENSE +15 -0
- scripts/__init__.py +2 -0
- scripts/clustering_3d.code-workspace +7 -0
- scripts/datasets/__init__.py +0 -0
- scripts/datasets/af_merge.py +344 -0
- scripts/datasets/build_datasets.py +125 -0
- scripts/datasets/get_pae.py +78 -0
- scripts/datasets/get_structures.py +107 -0
- scripts/datasets/model_confidence.py +97 -0
- scripts/datasets/parse_pae.py +64 -0
- scripts/datasets/prob_contact_maps.py +258 -0
- scripts/datasets/seq_for_mut_prob.py +900 -0
- scripts/datasets/utils.py +394 -0
- scripts/globals.py +169 -0
- scripts/main.py +650 -0
- scripts/plotting/__init__.py +0 -0
- scripts/plotting/build_annotations.py +102 -0
- scripts/plotting/chimerax_plot.py +251 -0
- scripts/plotting/pdb_tool.py +149 -0
- scripts/plotting/pfam.py +94 -0
- scripts/plotting/plot.py +2484 -0
- scripts/plotting/stability_change.py +184 -0
- scripts/plotting/uniprot_feat.py +276 -0
- scripts/plotting/utils.py +594 -0
- scripts/run/__init__.py +0 -0
- scripts/run/clustering.py +749 -0
- scripts/run/communities.py +53 -0
- scripts/run/miss_mut_prob.py +206 -0
- scripts/run/mutability.py +289 -0
- scripts/run/pvalues.py +91 -0
- scripts/run/score_and_simulations.py +155 -0
- scripts/run/utils.py +461 -0
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Module to merge overlapping fragments produced by AlphaFold 2
|
|
3
|
+
for the predictions of proteins larger than 2700 amino acids.
|
|
4
|
+
|
|
5
|
+
The module uses an adapted version of the code written by the
|
|
6
|
+
authors of DEGRONOPEDIA (Natalia A. Szulc, nszulc@iimcb.gov.pl).
|
|
7
|
+
DEGRONOPEDIA - a web server for proteome-wide inspection of degrons
|
|
8
|
+
doi: 10.1101/2022.05.19.492622.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import gzip
|
|
12
|
+
import logging
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
import shutil
|
|
16
|
+
import subprocess
|
|
17
|
+
from os import sep
|
|
18
|
+
|
|
19
|
+
import daiquiri
|
|
20
|
+
import pandas as pd
|
|
21
|
+
from Bio.PDB import PDBExceptions, PDBParser, Structure, Superimposer
|
|
22
|
+
from Bio.PDB.PDBIO import PDBIO
|
|
23
|
+
from tqdm import tqdm
|
|
24
|
+
|
|
25
|
+
from scripts import __logger_name__
|
|
26
|
+
|
|
27
|
+
logger = daiquiri.getLogger(__logger_name__ + ".build.af_merge")
|
|
28
|
+
|
|
29
|
+
daiquiri.getLogger('py.warnings').setLevel(logging.ERROR)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
## DEGRONOPEDIA script
|
|
33
|
+
|
|
34
|
+
def degronopedia_af_merge(struct_name, input_path, afold_version, output_path, zip):
|
|
35
|
+
"""
|
|
36
|
+
DEGRONOPEDIA script to merge any AlphaFold fragments into a unique structure.
|
|
37
|
+
|
|
38
|
+
DEGRONOPEDIA - a web server for proteome-wide inspection of degrons
|
|
39
|
+
doi: 10.1101/2022.05.19.492622
|
|
40
|
+
https://degronopedia.com/degronopedia/about
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
### ARGUMENTS PARSING ###
|
|
44
|
+
|
|
45
|
+
if input_path[-1] == sep:
|
|
46
|
+
input_path = input_path[:-1]
|
|
47
|
+
|
|
48
|
+
if output_path:
|
|
49
|
+
if output_path[-1] == sep:
|
|
50
|
+
output_path = output_path[:-1]
|
|
51
|
+
save_path = output_path
|
|
52
|
+
else:
|
|
53
|
+
save_path = input_path
|
|
54
|
+
|
|
55
|
+
### FIND OUT HOW MANY PIECES ###
|
|
56
|
+
|
|
57
|
+
how_many_pieces = 0
|
|
58
|
+
if zip:
|
|
59
|
+
onlyfiles = [f for f in os.listdir(input_path) if f.endswith(".pdb.gz") and os.path.isfile(os.path.join(input_path, f))]
|
|
60
|
+
else:
|
|
61
|
+
onlyfiles = [f for f in os.listdir(input_path) if f.endswith(".pdb") and os.path.isfile(os.path.join(input_path, f))]
|
|
62
|
+
|
|
63
|
+
for f in onlyfiles:
|
|
64
|
+
if struct_name in f and f[0] != '.': # do not include hidden files
|
|
65
|
+
how_many_pieces += 1
|
|
66
|
+
|
|
67
|
+
### MERGING ###
|
|
68
|
+
|
|
69
|
+
Bio_parser = PDBParser()
|
|
70
|
+
c = 1
|
|
71
|
+
while c < how_many_pieces :
|
|
72
|
+
|
|
73
|
+
struct_save_path = os.path.join(save_path, f"AF-{struct_name}-FM-model_v{afold_version}.pdb")
|
|
74
|
+
|
|
75
|
+
# Read reference structure
|
|
76
|
+
if c == 1:
|
|
77
|
+
struct_ref_path = os.path.join(input_path, f"AF-{struct_name}-F{c}-model_v{afold_version}.pdb")
|
|
78
|
+
if zip:
|
|
79
|
+
with gzip.open(f'{struct_ref_path}.gz', 'rt') as handle:
|
|
80
|
+
struct_ref = Bio_parser.get_structure("ref", handle)
|
|
81
|
+
else:
|
|
82
|
+
with open(struct_ref_path, 'r') as handle:
|
|
83
|
+
struct_ref = Bio_parser.get_structure("ref", handle)
|
|
84
|
+
else:
|
|
85
|
+
struct_ref = Bio_parser.get_structure("ref", struct_save_path)
|
|
86
|
+
model_ref = struct_ref[0]
|
|
87
|
+
|
|
88
|
+
# Read structure to superimpose
|
|
89
|
+
struct_si_path = os.path.join(input_path, f"AF-{struct_name}-F{c+1}-model_v{afold_version}.pdb")
|
|
90
|
+
if zip:
|
|
91
|
+
with gzip.open(f'{struct_si_path}.gz', 'rt') as handle:
|
|
92
|
+
structure_to_superpose = Bio_parser.get_structure("ref", handle)
|
|
93
|
+
else:
|
|
94
|
+
with open(struct_si_path, 'r') as handle:
|
|
95
|
+
structure_to_superpose = Bio_parser.get_structure("ref", handle)
|
|
96
|
+
model_to_super = structure_to_superpose[0]
|
|
97
|
+
|
|
98
|
+
# Append atoms from the nine last residues except for the very last one (it is C-end, has one more atom more)
|
|
99
|
+
model_ref_atoms = []
|
|
100
|
+
for j in range(len(model_ref['A'])-9, len(model_ref['A'])):
|
|
101
|
+
for atom in model_ref['A'][j]:
|
|
102
|
+
model_ref_atoms.append(atom)
|
|
103
|
+
|
|
104
|
+
# Append atoms from the 1191-1999 residues which correspond the abovementioned residues from the reference
|
|
105
|
+
model_to_superpose_atoms = []
|
|
106
|
+
for j in range(1191, 1200):
|
|
107
|
+
for atom in model_to_super['A'][j]:
|
|
108
|
+
model_to_superpose_atoms.append(atom)
|
|
109
|
+
|
|
110
|
+
# Superimpose
|
|
111
|
+
sup = Superimposer()
|
|
112
|
+
sup.set_atoms(model_ref_atoms, model_to_superpose_atoms)
|
|
113
|
+
|
|
114
|
+
# Update coords of the residues from the structure to be superimposed
|
|
115
|
+
sup.apply(model_to_super.get_atoms())
|
|
116
|
+
|
|
117
|
+
# Delete last residue (C-end residue, with one atom more) from the reference structure
|
|
118
|
+
model_ref['A'].detach_child((' ', len(model_ref['A']), ' '))
|
|
119
|
+
|
|
120
|
+
# Delete first 1199 residues from the superimposed structure
|
|
121
|
+
for i in range(1, 1200):
|
|
122
|
+
model_to_super['A'].detach_child((' ', i, ' '))
|
|
123
|
+
|
|
124
|
+
# Renumber residues in the superimposed structure
|
|
125
|
+
# Do it twice as you cannot assign a number to a residue that another residue already has
|
|
126
|
+
tmp_resnums = [i+1 for i in range(len(model_to_super['A']))]
|
|
127
|
+
|
|
128
|
+
for i, residue in enumerate(model_to_super['A'].get_residues()):
|
|
129
|
+
res_id = list(residue.id)
|
|
130
|
+
res_id[1] = tmp_resnums[i]
|
|
131
|
+
residue.id = tuple(res_id)
|
|
132
|
+
|
|
133
|
+
new_resnums = [i+len(model_ref['A'])+1 for i in range(len(model_to_super['A']))]
|
|
134
|
+
|
|
135
|
+
for i, residue in enumerate(model_to_super['A'].get_residues()):
|
|
136
|
+
res_id = list(residue.id)
|
|
137
|
+
res_id[1] = new_resnums[i]
|
|
138
|
+
residue.id = tuple(res_id)
|
|
139
|
+
|
|
140
|
+
# Merge and save both structures however as two models
|
|
141
|
+
merged = Structure.Structure("master")
|
|
142
|
+
merged.add(model_ref)
|
|
143
|
+
model_to_super.id='B'
|
|
144
|
+
merged.add(model_to_super)
|
|
145
|
+
|
|
146
|
+
io = PDBIO()
|
|
147
|
+
io.set_structure(merged)
|
|
148
|
+
io.save(struct_save_path)
|
|
149
|
+
|
|
150
|
+
# Unify models
|
|
151
|
+
bashCommand1 = os.path.join("sed '", "TER", f"d' {save_path}", f"AF-{struct_name}-FM-model_v{afold_version}.pdb > {save_path}", "tmp.pdb")
|
|
152
|
+
bashCommand2 = os.path.join("sed '", "MODEL", f"d' {save_path}", f"tmp.pdb > {save_path}", "tmp1.pdb")
|
|
153
|
+
bashCommand3 = os.path.join("sed '", "ENDMDL", f"d' {save_path}", f"tmp1.pdb > {save_path}", "tmp2.pdb")
|
|
154
|
+
|
|
155
|
+
subprocess.run(bashCommand1, check=True, text=True, shell=True)
|
|
156
|
+
subprocess.run(bashCommand2, check=True, text=True, shell=True)
|
|
157
|
+
subprocess.run(bashCommand3, check=True, text=True, shell=True)
|
|
158
|
+
|
|
159
|
+
# Re-read the structure in Biopython and save
|
|
160
|
+
structure_ok = Bio_parser.get_structure("ok", os.path.join(save_path, "tmp2.pdb"))
|
|
161
|
+
io.set_structure(structure_ok)
|
|
162
|
+
io.save(struct_save_path)
|
|
163
|
+
|
|
164
|
+
c += 1
|
|
165
|
+
|
|
166
|
+
# Add MODEL 1 at the beggining of the file
|
|
167
|
+
bashCommand4 = os.path.join(f"sed -i '1iMODEL 1 ' {save_path}", f"AF-{struct_name}-FM-model_v{afold_version}.pdb")
|
|
168
|
+
subprocess.run(bashCommand4, check=True, shell=True)
|
|
169
|
+
|
|
170
|
+
# Provide proper file ending
|
|
171
|
+
bashCommand5 = os.path.join(f"sed -i '$ d' {save_path}", f"AF-{struct_name}-FM-model_v{afold_version}.pdb")
|
|
172
|
+
bashCommand6 = os.path.join(f"echo 'ENDMDL ' >> {save_path}", f"AF-{struct_name}-FM-model_v{afold_version}.pdb")
|
|
173
|
+
bashCommand7 = os.path.join(f"echo 'END ' >> {save_path}", f"AF-{struct_name}-FM-model_v{afold_version}.pdb")
|
|
174
|
+
subprocess.run(bashCommand5, check=True, shell=True)
|
|
175
|
+
subprocess.run(bashCommand6, check=True, text=True, shell=True)
|
|
176
|
+
subprocess.run(bashCommand7, check=True, text=True, shell=True)
|
|
177
|
+
|
|
178
|
+
# Delete tmp files
|
|
179
|
+
bashCommand8 = os.path.join(f"rm {save_path}", f"tmp.pdb {save_path}", f"tmp1.pdb {save_path}", "tmp2.pdb")
|
|
180
|
+
subprocess.run(bashCommand8, check=True, text=True, shell=True)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
## In-house scripts
|
|
184
|
+
|
|
185
|
+
# Add SEQREF record to pdb file
|
|
186
|
+
|
|
187
|
+
def get_res_from_chain(pdb_path):
|
|
188
|
+
"""
|
|
189
|
+
Get sequense of amino acid residues from the structure chain.
|
|
190
|
+
"""
|
|
191
|
+
|
|
192
|
+
# Load structure
|
|
193
|
+
parser = PDBParser()
|
|
194
|
+
structure = parser.get_structure("ID", pdb_path)
|
|
195
|
+
|
|
196
|
+
# Get seq from chain
|
|
197
|
+
residues = []
|
|
198
|
+
chain = structure[0]["A"]
|
|
199
|
+
for residue in chain.get_residues():
|
|
200
|
+
residues.append(residue.resname)
|
|
201
|
+
|
|
202
|
+
return residues
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def get_pdb_seqres_records(lst_res):
|
|
206
|
+
"""
|
|
207
|
+
Construct the fixed-width records of a pdb file.
|
|
208
|
+
"""
|
|
209
|
+
|
|
210
|
+
records = []
|
|
211
|
+
num_residues = len(lst_res)
|
|
212
|
+
record_counter = 0
|
|
213
|
+
while record_counter * 13 < num_residues:
|
|
214
|
+
start_idx = record_counter * 13
|
|
215
|
+
end_idx = min(start_idx + 13, num_residues)
|
|
216
|
+
residue_subset = lst_res[start_idx:end_idx]
|
|
217
|
+
record = 'SEQRES {:>3} {} {:>4} {:52}\n'.format(record_counter+1, "A", num_residues, ' '.join(residue_subset))
|
|
218
|
+
records.append(record)
|
|
219
|
+
record_counter += 1
|
|
220
|
+
|
|
221
|
+
return records
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def add_refseq_record_to_pdb(path_structure):
|
|
225
|
+
"""
|
|
226
|
+
Add the SEQREF records to the pdb file.
|
|
227
|
+
"""
|
|
228
|
+
|
|
229
|
+
# Open the PDB file and get SEQRES insert index
|
|
230
|
+
with open(path_structure, 'r') as file:
|
|
231
|
+
pdb_lines = file.readlines()
|
|
232
|
+
insert_index = next(i for i, line in enumerate(pdb_lines) if line.startswith('MODEL'))
|
|
233
|
+
|
|
234
|
+
# Get seares records
|
|
235
|
+
residues = get_res_from_chain(path_structure)
|
|
236
|
+
seqres_records = get_pdb_seqres_records(residues)
|
|
237
|
+
|
|
238
|
+
# Insert the SEQRES records
|
|
239
|
+
pdb_lines[insert_index:insert_index] = seqres_records
|
|
240
|
+
|
|
241
|
+
# Save
|
|
242
|
+
with open(path_structure, 'w') as output_file:
|
|
243
|
+
output_file.truncate()
|
|
244
|
+
output_file.writelines(pdb_lines)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
# Other functions
|
|
248
|
+
|
|
249
|
+
def get_list_fragmented_pdb(pdb_dir):
|
|
250
|
+
"""
|
|
251
|
+
Given a directory including pdb files, return a list of tuples (Uniprot_ID, max AF_F).
|
|
252
|
+
"""
|
|
253
|
+
|
|
254
|
+
# List pdb files
|
|
255
|
+
list_pdb = os.listdir(pdb_dir)
|
|
256
|
+
list_pdb = [file for file in list_pdb if not file.startswith("tmp") and file.endswith(".pdb") or file.endswith(".pdb.gz")]
|
|
257
|
+
list_pdb = [(file.split("-")[1], re.sub(r"\D", "", file.split("-")[2])) for file in list_pdb if file.split("-")[2][-1] != "M"]
|
|
258
|
+
|
|
259
|
+
# Get df with max fragment
|
|
260
|
+
df = pd.DataFrame(list_pdb, columns=["Uniprot_ID", "F"])
|
|
261
|
+
df["F"] = pd.to_numeric(df["F"])
|
|
262
|
+
df = df.groupby("Uniprot_ID").max()
|
|
263
|
+
|
|
264
|
+
# Get fragmented structures as list of (Uniprot_ID AF_F) tuples
|
|
265
|
+
df = df[df["F"] > 1].reset_index()
|
|
266
|
+
|
|
267
|
+
return list(df.to_records(index=False))
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def save_unprocessed_ids(uni_ids, filename):
|
|
271
|
+
|
|
272
|
+
with open(filename, 'a') as file:
|
|
273
|
+
for id in uni_ids:
|
|
274
|
+
file.write(id + '\n')
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
# Wrapper function
|
|
278
|
+
|
|
279
|
+
def merge_af_fragments(input_dir, output_dir=None, af_version=4, gzip=False):
|
|
280
|
+
"""
|
|
281
|
+
Run and parse DEGRONOPEDIA script to merge any AlphaFold fragments into a unique structure.
|
|
282
|
+
|
|
283
|
+
DEGRONOPEDIA - a web server for proteome-wide inspection of degrons
|
|
284
|
+
doi: 10.1101/2022.05.19.492622
|
|
285
|
+
https://degronopedia.com/degronopedia/about
|
|
286
|
+
"""
|
|
287
|
+
|
|
288
|
+
if output_dir is None:
|
|
289
|
+
output_dir = input_dir
|
|
290
|
+
if gzip:
|
|
291
|
+
zip_ext = ".gz"
|
|
292
|
+
else:
|
|
293
|
+
zip_ext = ""
|
|
294
|
+
|
|
295
|
+
fragments = get_list_fragmented_pdb(input_dir)
|
|
296
|
+
if len(fragments) > 0:
|
|
297
|
+
|
|
298
|
+
# Create dir where to move original fragmented structures
|
|
299
|
+
path_original_frag = os.path.join(output_dir, "fragmented_pdbs")
|
|
300
|
+
if not os.path.exists(path_original_frag):
|
|
301
|
+
os.makedirs(path_original_frag)
|
|
302
|
+
checkpoint = os.path.join(path_original_frag, '.checkpoint.merge.txt')
|
|
303
|
+
|
|
304
|
+
if os.path.exists(checkpoint):
|
|
305
|
+
logger.debug("Merge fragments already performed: Skipping...")
|
|
306
|
+
else:
|
|
307
|
+
# Get list of fragmented Uniprot ID and max AF-F
|
|
308
|
+
not_processed = []
|
|
309
|
+
for uni_id, max_f in tqdm(fragments, total=len(fragments), desc="Merging AF fragments"):
|
|
310
|
+
|
|
311
|
+
processed = False
|
|
312
|
+
|
|
313
|
+
try:
|
|
314
|
+
degronopedia_af_merge(uni_id, input_dir, af_version, output_dir, gzip)
|
|
315
|
+
processed = True
|
|
316
|
+
except (PDBExceptions.PDBIOException, PDBExceptions.PDBConstructionException):
|
|
317
|
+
logger.warning(f"Could not process {uni_id} ({max_f} fragments)")
|
|
318
|
+
not_processed.append(uni_id)
|
|
319
|
+
f_path = os.path.join(output_dir, f"AF-{uni_id}-FM-model_v{af_version}.pdb")
|
|
320
|
+
if os.path.exists(f_path):
|
|
321
|
+
os.remove(f_path)
|
|
322
|
+
# Move the original fragmented structures
|
|
323
|
+
for f in range(1, max_f+1):
|
|
324
|
+
file = f"AF-{uni_id}-F{f}-model_v{af_version}.pdb{zip_ext}"
|
|
325
|
+
shutil.move(os.path.join(input_dir, file), path_original_frag)
|
|
326
|
+
|
|
327
|
+
# Rename merged structure and add refseq records to pdb
|
|
328
|
+
if processed:
|
|
329
|
+
tmp_name = os.path.join(output_dir, f"AF-{uni_id}-FM-model_v{af_version}.pdb")
|
|
330
|
+
name = os.path.join(output_dir, f"AF-{uni_id}-F{max_f}M-model_v{af_version}.pdb")
|
|
331
|
+
os.rename(tmp_name, name)
|
|
332
|
+
add_refseq_record_to_pdb(name)
|
|
333
|
+
|
|
334
|
+
if len(not_processed) > 0:
|
|
335
|
+
logger.warning(f"Not processed: {not_processed}")
|
|
336
|
+
with open(checkpoint, "w") as f:
|
|
337
|
+
f.write('')
|
|
338
|
+
|
|
339
|
+
save_unprocessed_ids(not_processed,
|
|
340
|
+
os.path.join(output_dir, "fragmented_pdbs", "ids_not_merged.txt"))
|
|
341
|
+
logger.info("Merge of structures completed!")
|
|
342
|
+
|
|
343
|
+
else:
|
|
344
|
+
logger.debug("Nothing to merge: Skipping...")
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Module to generate datasets necessary to run Oncodrive3D.
|
|
3
|
+
|
|
4
|
+
The build is a pipeline that perform the following tasks:
|
|
5
|
+
- Download the PDB structures of the selected proteome
|
|
6
|
+
predicted by AlphaFold 2 from AlphaFold DB.
|
|
7
|
+
- Merge the overlapping structures processed as fragments.
|
|
8
|
+
- Extract AlphaFold model confidence (pLDDT).
|
|
9
|
+
- Generate a dataframe including Uniprot_ID, HUGO Symbol,
|
|
10
|
+
protein, DNA sequence, and other gene's information.
|
|
11
|
+
- Download AlphaFold predicted aligned error (PAE) from
|
|
12
|
+
AlphaFold DB and convert the files into npy format.
|
|
13
|
+
- Use the PDB structure and PAE to create maps of
|
|
14
|
+
probability of contacts (pCMAPs) for any protein of the
|
|
15
|
+
downloaded proteome with available PAE.
|
|
16
|
+
- Remove unnecessary temp files.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
import os
|
|
21
|
+
|
|
22
|
+
import daiquiri
|
|
23
|
+
|
|
24
|
+
from scripts import __logger_name__
|
|
25
|
+
from scripts.datasets.af_merge import merge_af_fragments
|
|
26
|
+
from scripts.datasets.get_pae import get_pae
|
|
27
|
+
from scripts.datasets.get_structures import get_structures, mv_mane_pdb
|
|
28
|
+
from scripts.datasets.model_confidence import get_confidence
|
|
29
|
+
from scripts.datasets.parse_pae import parse_pae
|
|
30
|
+
from scripts.datasets.prob_contact_maps import get_prob_cmaps_mp
|
|
31
|
+
from scripts.datasets.seq_for_mut_prob import get_seq_df
|
|
32
|
+
from scripts.datasets.utils import get_species
|
|
33
|
+
from scripts.globals import clean_dir, clean_temp_files
|
|
34
|
+
|
|
35
|
+
logger = daiquiri.getLogger(__logger_name__ + ".build")
|
|
36
|
+
|
|
37
|
+
def build(output_datasets,
|
|
38
|
+
organism,
|
|
39
|
+
mane,
|
|
40
|
+
distance_threshold,
|
|
41
|
+
num_cores,
|
|
42
|
+
af_version,
|
|
43
|
+
mane_version):
|
|
44
|
+
"""
|
|
45
|
+
Build datasets necessary to run Oncodrive3D.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
# Empty directory
|
|
49
|
+
clean_dir(output_datasets, 'd', txt_file=True)
|
|
50
|
+
|
|
51
|
+
# Download PDB structures
|
|
52
|
+
species = get_species(organism)
|
|
53
|
+
logger.info("Downloading AlphaFold (AF) predicted structures...")
|
|
54
|
+
get_structures(path=os.path.join(output_datasets,"pdb_structures"),
|
|
55
|
+
species=species,
|
|
56
|
+
af_version=str(af_version),
|
|
57
|
+
threads=num_cores)
|
|
58
|
+
logger.info("Download of structures completed!")
|
|
59
|
+
|
|
60
|
+
# Merge fragmented structures
|
|
61
|
+
logger.info("Merging fragmented structures...")
|
|
62
|
+
merge_af_fragments(input_dir=os.path.join(output_datasets,"pdb_structures"),
|
|
63
|
+
gzip=True)
|
|
64
|
+
|
|
65
|
+
# Download PDB MANE structures
|
|
66
|
+
if species == "Homo sapiens" and mane == True:
|
|
67
|
+
logger.info("Downloading AlphaFold (AF) predicted structures overlap with MANE...")
|
|
68
|
+
get_structures(path=os.path.join(output_datasets,"pdb_structures_mane"),
|
|
69
|
+
species=species,
|
|
70
|
+
mane=True,
|
|
71
|
+
threads=num_cores)
|
|
72
|
+
mv_mane_pdb(output_datasets, "pdb_structures", "pdb_structures_mane")
|
|
73
|
+
logger.info("Download of MANE structures completed!")
|
|
74
|
+
|
|
75
|
+
# Create df including genes and proteins sequences & Hugo to Uniprot_ID mapping
|
|
76
|
+
logger.info("Generating dataframe for genes and proteins sequences...")
|
|
77
|
+
seq_df = get_seq_df(datasets_dir=output_datasets,
|
|
78
|
+
output_seq_df=os.path.join(output_datasets, "seq_for_mut_prob.tsv"),
|
|
79
|
+
organism=species,
|
|
80
|
+
mane=mane,
|
|
81
|
+
num_cores=num_cores,
|
|
82
|
+
mane_version=mane_version)
|
|
83
|
+
logger.info("Generation of sequences dataframe completed!")
|
|
84
|
+
|
|
85
|
+
# Get model confidence
|
|
86
|
+
logger.info("Extracting AF model confidence...")
|
|
87
|
+
get_confidence(input=os.path.join(output_datasets, "pdb_structures"),
|
|
88
|
+
output_dir=os.path.join(output_datasets),
|
|
89
|
+
seq_df=seq_df)
|
|
90
|
+
|
|
91
|
+
# Get PAE
|
|
92
|
+
logger.info("Downloading AF predicted aligned error (PAE)...")
|
|
93
|
+
get_pae(input_dir=os.path.join(output_datasets,"pdb_structures"),
|
|
94
|
+
output_dir=os.path.join(output_datasets,"pae"),
|
|
95
|
+
num_cores=num_cores,
|
|
96
|
+
af_version=str(af_version))
|
|
97
|
+
|
|
98
|
+
# Parse PAE
|
|
99
|
+
logger.info("Parsing PAE...")
|
|
100
|
+
parse_pae(input=os.path.join(output_datasets, 'pae'))
|
|
101
|
+
logger.info("Parsing PAE completed!")
|
|
102
|
+
|
|
103
|
+
# Get pCAMPs
|
|
104
|
+
logger.info("Generating contact probability maps (pCMAPs)..")
|
|
105
|
+
get_prob_cmaps_mp(input_pdb=os.path.join(output_datasets, "pdb_structures"),
|
|
106
|
+
input_pae=os.path.join(output_datasets, "pae"),
|
|
107
|
+
output=os.path.join(output_datasets,"prob_cmaps"),
|
|
108
|
+
distance=distance_threshold,
|
|
109
|
+
num_cores=num_cores)
|
|
110
|
+
logger.info("Generation pCMAPs completed!")
|
|
111
|
+
|
|
112
|
+
# Clean datasets
|
|
113
|
+
logger.info("Cleaning datasets...")
|
|
114
|
+
clean_temp_files(path=output_datasets)
|
|
115
|
+
logger.info("Datasets cleaning completed!")
|
|
116
|
+
logger.info("Datasets have been successfully built and are ready for analysis!")
|
|
117
|
+
|
|
118
|
+
if __name__ == "__main__":
|
|
119
|
+
build(output_datasets="/workspace/nobackup/scratch/oncodrive3d/datasets_mane",
|
|
120
|
+
organism="Homo sapiens",
|
|
121
|
+
mane=True,
|
|
122
|
+
distance_threshold=10,
|
|
123
|
+
num_cores=8,
|
|
124
|
+
af_version=4,
|
|
125
|
+
mane_version=1.3)
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
import daiquiri
|
|
4
|
+
import requests
|
|
5
|
+
import time
|
|
6
|
+
import concurrent.futures
|
|
7
|
+
from tqdm import tqdm
|
|
8
|
+
|
|
9
|
+
from scripts import __logger_name__
|
|
10
|
+
|
|
11
|
+
logger = daiquiri.getLogger(__logger_name__ + ".build.PAE")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def download_pae(uniprot_id: str, af_version: int, output_dir: str) -> None:
|
|
15
|
+
"""
|
|
16
|
+
Download Predicted Aligned Error (PAE) file from AlphaFold DB.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
uniprot_id (str): Uniprot ID of the structure.
|
|
20
|
+
af_version (int): AlphaFold 2 version.
|
|
21
|
+
output_dir (str): Output directory where to download the PAE files.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
file_path = os.path.join(output_dir, f"{uniprot_id}-F1-predicted_aligned_error.json")
|
|
25
|
+
download_url = f"https://alphafold.ebi.ac.uk/files/AF-{uniprot_id}-F1-predicted_aligned_error_v{af_version}.json"
|
|
26
|
+
|
|
27
|
+
i = 0
|
|
28
|
+
status = "INIT"
|
|
29
|
+
while status != "FINISHED":
|
|
30
|
+
i += 1
|
|
31
|
+
if status != "INIT":
|
|
32
|
+
time.sleep(30)
|
|
33
|
+
try:
|
|
34
|
+
response = requests.get(download_url, timeout=30)
|
|
35
|
+
content = response.content
|
|
36
|
+
if content.endswith(b'}]') and not content.endswith(b'</Error>'):
|
|
37
|
+
with open(file_path, 'wb') as output_file:
|
|
38
|
+
output_file.write(content)
|
|
39
|
+
status = "FINISHED"
|
|
40
|
+
except requests.exceptions.RequestException as e:
|
|
41
|
+
status = "ERROR"
|
|
42
|
+
if i % 10 == 0:
|
|
43
|
+
logger.debug(f"Request failed {e}: Retrying")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def get_pae(input_dir: str, output_dir: str, num_cores: int, af_version: int = 4) -> None:
|
|
47
|
+
"""
|
|
48
|
+
Download Predicted Aligned Error (PAE) files for all non-fragmented PDB
|
|
49
|
+
structures in the input directory.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
input_dir (str): Input directory including the PDB structures.
|
|
53
|
+
output_dir (str): Output directory where to download the PAE files.
|
|
54
|
+
num_cores (int): Number of cores for multithreading download.
|
|
55
|
+
af_version (int): AlphaFold 2 version (default is 4).
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
if not os.path.exists(output_dir):
|
|
59
|
+
os.makedirs(output_dir)
|
|
60
|
+
|
|
61
|
+
checkpoint = os.path.join(output_dir, '.checkpoint.txt')
|
|
62
|
+
if os.path.exists(checkpoint):
|
|
63
|
+
logger.debug("PAE already downloaded: Skipping...")
|
|
64
|
+
return
|
|
65
|
+
|
|
66
|
+
pdb_files = [file for file in os.listdir(input_dir) if file.startswith("AF-") and file.endswith(f"-model_v{af_version}.pdb.gz")]
|
|
67
|
+
uniprot_ids = [pdb_file.split("-")[1] for pdb_file in pdb_files]
|
|
68
|
+
|
|
69
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=num_cores) as executor:
|
|
70
|
+
tasks = [executor.submit(download_pae, uniprot_id, af_version, output_dir) for uniprot_id in uniprot_ids]
|
|
71
|
+
|
|
72
|
+
for _ in tqdm(concurrent.futures.as_completed(tasks), total=len(tasks), desc="Downloading PAE"):
|
|
73
|
+
pass
|
|
74
|
+
|
|
75
|
+
with open(checkpoint, "w") as f:
|
|
76
|
+
f.write('')
|
|
77
|
+
|
|
78
|
+
logger.info('Download of PAE completed!')
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
import daiquiri
|
|
5
|
+
import time
|
|
6
|
+
import subprocess
|
|
7
|
+
import shutil
|
|
8
|
+
|
|
9
|
+
from scripts import __logger_name__
|
|
10
|
+
from scripts.datasets.utils import calculate_hash, download_single_file, extract_tar_file, assert_proteome_integrity, CHECKSUM
|
|
11
|
+
|
|
12
|
+
logger = daiquiri.getLogger(__logger_name__ + ".build.AF-pdb")
|
|
13
|
+
|
|
14
|
+
logging.getLogger('urllib3.connectionpool').setLevel(logging.WARNING)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def mv_mane_pdb(path_datasets, pdb_dir, mane_pdb_dir) -> None:
|
|
18
|
+
"""
|
|
19
|
+
Move AF structures with overlap with MANE Select
|
|
20
|
+
transcripts to directory with AF structures from
|
|
21
|
+
human proteome. Overwrite any overlapping AF ID.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
path_pdb = os.path.join(path_datasets, pdb_dir)
|
|
25
|
+
path_mane_pdb = os.path.join(path_datasets, mane_pdb_dir)
|
|
26
|
+
if not os.path.exists(path_mane_pdb):
|
|
27
|
+
os.makedirs(path_mane_pdb)
|
|
28
|
+
|
|
29
|
+
# Move MANE structures
|
|
30
|
+
for filename in [file for file in os.listdir(path_mane_pdb) if file.endswith(".pdb.gz") or file.endswith(".pdb")]:
|
|
31
|
+
pdb = os.path.join(path_pdb, filename)
|
|
32
|
+
pdb_mane = os.path.join(path_mane_pdb, filename)
|
|
33
|
+
shutil.move(pdb_mane, pdb)
|
|
34
|
+
|
|
35
|
+
# Move MANE metadata files
|
|
36
|
+
for filename in [file for file in os.listdir(os.path.join(path_mane_pdb)) if file.endswith(".csv") or file.endswith("readme.txt")]:
|
|
37
|
+
source_file = os.path.join(path_mane_pdb, filename)
|
|
38
|
+
dest_file = os.path.join(path_datasets, f"mane_{filename}")
|
|
39
|
+
shutil.move(source_file, dest_file)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def get_structures(path: str,
|
|
43
|
+
species: str = 'Homo sapiens',
|
|
44
|
+
mane: bool = False,
|
|
45
|
+
af_version: str = '4',
|
|
46
|
+
threads: int = 1,
|
|
47
|
+
max_attempts: int = 30) -> None:
|
|
48
|
+
"""
|
|
49
|
+
Downloads AlphaFold predicted structures for a given organism and version.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
path (str): Path where to save PDB structures.
|
|
53
|
+
species (str, optional): Species (human (default) or mouse). Defaults to 'human'.
|
|
54
|
+
af_version (str, optional): AlphaFold 2 version (4 as default). Defaults to '4'.
|
|
55
|
+
verbose (str, optional): Verbose (True (default) or False). Defaults to 'False'.
|
|
56
|
+
|
|
57
|
+
Example:
|
|
58
|
+
get_structures('datasets/pdb_structures', species='human', verbose='True')
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
logger.info(f"Selected species: {species}")
|
|
62
|
+
|
|
63
|
+
if not os.path.isdir(path):
|
|
64
|
+
os.makedirs(path)
|
|
65
|
+
logger.debug(f'mkdir {path}')
|
|
66
|
+
|
|
67
|
+
# Select proteome
|
|
68
|
+
if mane:
|
|
69
|
+
if species == "Homo sapiens":
|
|
70
|
+
proteome = f"mane_overlap_v{af_version}"
|
|
71
|
+
else:
|
|
72
|
+
raise RuntimeError(f"Structures with MANE transcripts overlap are available only for 'Homo sapiens'. Exiting...")
|
|
73
|
+
else:
|
|
74
|
+
if species == "Homo sapiens":
|
|
75
|
+
proteome = f"UP000005640_9606_HUMAN_v{af_version}"
|
|
76
|
+
elif species == "Mus musculus":
|
|
77
|
+
proteome = f"UP000000589_10090_MOUSE_v{af_version}"
|
|
78
|
+
else:
|
|
79
|
+
raise RuntimeError(f"Failed to recognize '{species}' as organism. Currently accepted ones are 'Homo sapiens' and 'Mus musculus'. Exiting...")
|
|
80
|
+
|
|
81
|
+
logger.debug(f"Proteome to download: {proteome}")
|
|
82
|
+
af_url = f"https://ftp.ebi.ac.uk/pub/databases/alphafold/latest/{proteome}.tar"
|
|
83
|
+
file_path = os.path.join(path, f"{proteome}.tar")
|
|
84
|
+
|
|
85
|
+
try:
|
|
86
|
+
## STEP1 --- Download file
|
|
87
|
+
attempts = 0
|
|
88
|
+
status = "INIT"
|
|
89
|
+
while status != "PASS":
|
|
90
|
+
download_single_file(af_url, file_path, threads, proteome)
|
|
91
|
+
status = assert_proteome_integrity(file_path, proteome)
|
|
92
|
+
attempts += 1
|
|
93
|
+
if attempts >= max_attempts:
|
|
94
|
+
raise RuntimeError(f"Failed to download with integrity after {max_attempts} attempts. Exiting...")
|
|
95
|
+
time.sleep(10)
|
|
96
|
+
|
|
97
|
+
## STEP2 --- Extract structures
|
|
98
|
+
logger.info(f'Extracting {file_path}')
|
|
99
|
+
extract_tar_file(file_path, path)
|
|
100
|
+
|
|
101
|
+
logger.info('Download structure: SUCCESS')
|
|
102
|
+
logger.debug(f"Structures downloaded in directory {path}")
|
|
103
|
+
|
|
104
|
+
except Exception as e:
|
|
105
|
+
logger.error('Download structure: FAIL')
|
|
106
|
+
logger.error(f"Error while downloading structures: {e}")
|
|
107
|
+
raise e
|