Oncodrive3D 1.0.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,344 @@
1
+ """
2
+ Module to merge overlapping fragments produced by AlphaFold 2
3
+ for the predictions of proteins larger than 2700 amino acids.
4
+
5
+ The module uses an adapted version of the code written by the
6
+ authors of DEGRONOPEDIA (Natalia A. Szulc, nszulc@iimcb.gov.pl).
7
+ DEGRONOPEDIA - a web server for proteome-wide inspection of degrons
8
+ doi: 10.1101/2022.05.19.492622.
9
+ """
10
+
11
+ import gzip
12
+ import logging
13
+ import os
14
+ import re
15
+ import shutil
16
+ import subprocess
17
+ from os import sep
18
+
19
+ import daiquiri
20
+ import pandas as pd
21
+ from Bio.PDB import PDBExceptions, PDBParser, Structure, Superimposer
22
+ from Bio.PDB.PDBIO import PDBIO
23
+ from tqdm import tqdm
24
+
25
+ from scripts import __logger_name__
26
+
27
+ logger = daiquiri.getLogger(__logger_name__ + ".build.af_merge")
28
+
29
+ daiquiri.getLogger('py.warnings').setLevel(logging.ERROR)
30
+
31
+
32
+ ## DEGRONOPEDIA script
33
+
34
+ def degronopedia_af_merge(struct_name, input_path, afold_version, output_path, zip):
35
+ """
36
+ DEGRONOPEDIA script to merge any AlphaFold fragments into a unique structure.
37
+
38
+ DEGRONOPEDIA - a web server for proteome-wide inspection of degrons
39
+ doi: 10.1101/2022.05.19.492622
40
+ https://degronopedia.com/degronopedia/about
41
+ """
42
+
43
+ ### ARGUMENTS PARSING ###
44
+
45
+ if input_path[-1] == sep:
46
+ input_path = input_path[:-1]
47
+
48
+ if output_path:
49
+ if output_path[-1] == sep:
50
+ output_path = output_path[:-1]
51
+ save_path = output_path
52
+ else:
53
+ save_path = input_path
54
+
55
+ ### FIND OUT HOW MANY PIECES ###
56
+
57
+ how_many_pieces = 0
58
+ if zip:
59
+ onlyfiles = [f for f in os.listdir(input_path) if f.endswith(".pdb.gz") and os.path.isfile(os.path.join(input_path, f))]
60
+ else:
61
+ onlyfiles = [f for f in os.listdir(input_path) if f.endswith(".pdb") and os.path.isfile(os.path.join(input_path, f))]
62
+
63
+ for f in onlyfiles:
64
+ if struct_name in f and f[0] != '.': # do not include hidden files
65
+ how_many_pieces += 1
66
+
67
+ ### MERGING ###
68
+
69
+ Bio_parser = PDBParser()
70
+ c = 1
71
+ while c < how_many_pieces :
72
+
73
+ struct_save_path = os.path.join(save_path, f"AF-{struct_name}-FM-model_v{afold_version}.pdb")
74
+
75
+ # Read reference structure
76
+ if c == 1:
77
+ struct_ref_path = os.path.join(input_path, f"AF-{struct_name}-F{c}-model_v{afold_version}.pdb")
78
+ if zip:
79
+ with gzip.open(f'{struct_ref_path}.gz', 'rt') as handle:
80
+ struct_ref = Bio_parser.get_structure("ref", handle)
81
+ else:
82
+ with open(struct_ref_path, 'r') as handle:
83
+ struct_ref = Bio_parser.get_structure("ref", handle)
84
+ else:
85
+ struct_ref = Bio_parser.get_structure("ref", struct_save_path)
86
+ model_ref = struct_ref[0]
87
+
88
+ # Read structure to superimpose
89
+ struct_si_path = os.path.join(input_path, f"AF-{struct_name}-F{c+1}-model_v{afold_version}.pdb")
90
+ if zip:
91
+ with gzip.open(f'{struct_si_path}.gz', 'rt') as handle:
92
+ structure_to_superpose = Bio_parser.get_structure("ref", handle)
93
+ else:
94
+ with open(struct_si_path, 'r') as handle:
95
+ structure_to_superpose = Bio_parser.get_structure("ref", handle)
96
+ model_to_super = structure_to_superpose[0]
97
+
98
+ # Append atoms from the nine last residues except for the very last one (it is C-end, has one more atom more)
99
+ model_ref_atoms = []
100
+ for j in range(len(model_ref['A'])-9, len(model_ref['A'])):
101
+ for atom in model_ref['A'][j]:
102
+ model_ref_atoms.append(atom)
103
+
104
+ # Append atoms from the 1191-1999 residues which correspond the abovementioned residues from the reference
105
+ model_to_superpose_atoms = []
106
+ for j in range(1191, 1200):
107
+ for atom in model_to_super['A'][j]:
108
+ model_to_superpose_atoms.append(atom)
109
+
110
+ # Superimpose
111
+ sup = Superimposer()
112
+ sup.set_atoms(model_ref_atoms, model_to_superpose_atoms)
113
+
114
+ # Update coords of the residues from the structure to be superimposed
115
+ sup.apply(model_to_super.get_atoms())
116
+
117
+ # Delete last residue (C-end residue, with one atom more) from the reference structure
118
+ model_ref['A'].detach_child((' ', len(model_ref['A']), ' '))
119
+
120
+ # Delete first 1199 residues from the superimposed structure
121
+ for i in range(1, 1200):
122
+ model_to_super['A'].detach_child((' ', i, ' '))
123
+
124
+ # Renumber residues in the superimposed structure
125
+ # Do it twice as you cannot assign a number to a residue that another residue already has
126
+ tmp_resnums = [i+1 for i in range(len(model_to_super['A']))]
127
+
128
+ for i, residue in enumerate(model_to_super['A'].get_residues()):
129
+ res_id = list(residue.id)
130
+ res_id[1] = tmp_resnums[i]
131
+ residue.id = tuple(res_id)
132
+
133
+ new_resnums = [i+len(model_ref['A'])+1 for i in range(len(model_to_super['A']))]
134
+
135
+ for i, residue in enumerate(model_to_super['A'].get_residues()):
136
+ res_id = list(residue.id)
137
+ res_id[1] = new_resnums[i]
138
+ residue.id = tuple(res_id)
139
+
140
+ # Merge and save both structures however as two models
141
+ merged = Structure.Structure("master")
142
+ merged.add(model_ref)
143
+ model_to_super.id='B'
144
+ merged.add(model_to_super)
145
+
146
+ io = PDBIO()
147
+ io.set_structure(merged)
148
+ io.save(struct_save_path)
149
+
150
+ # Unify models
151
+ bashCommand1 = os.path.join("sed '", "TER", f"d' {save_path}", f"AF-{struct_name}-FM-model_v{afold_version}.pdb > {save_path}", "tmp.pdb")
152
+ bashCommand2 = os.path.join("sed '", "MODEL", f"d' {save_path}", f"tmp.pdb > {save_path}", "tmp1.pdb")
153
+ bashCommand3 = os.path.join("sed '", "ENDMDL", f"d' {save_path}", f"tmp1.pdb > {save_path}", "tmp2.pdb")
154
+
155
+ subprocess.run(bashCommand1, check=True, text=True, shell=True)
156
+ subprocess.run(bashCommand2, check=True, text=True, shell=True)
157
+ subprocess.run(bashCommand3, check=True, text=True, shell=True)
158
+
159
+ # Re-read the structure in Biopython and save
160
+ structure_ok = Bio_parser.get_structure("ok", os.path.join(save_path, "tmp2.pdb"))
161
+ io.set_structure(structure_ok)
162
+ io.save(struct_save_path)
163
+
164
+ c += 1
165
+
166
+ # Add MODEL 1 at the beggining of the file
167
+ bashCommand4 = os.path.join(f"sed -i '1iMODEL 1 ' {save_path}", f"AF-{struct_name}-FM-model_v{afold_version}.pdb")
168
+ subprocess.run(bashCommand4, check=True, shell=True)
169
+
170
+ # Provide proper file ending
171
+ bashCommand5 = os.path.join(f"sed -i '$ d' {save_path}", f"AF-{struct_name}-FM-model_v{afold_version}.pdb")
172
+ bashCommand6 = os.path.join(f"echo 'ENDMDL ' >> {save_path}", f"AF-{struct_name}-FM-model_v{afold_version}.pdb")
173
+ bashCommand7 = os.path.join(f"echo 'END ' >> {save_path}", f"AF-{struct_name}-FM-model_v{afold_version}.pdb")
174
+ subprocess.run(bashCommand5, check=True, shell=True)
175
+ subprocess.run(bashCommand6, check=True, text=True, shell=True)
176
+ subprocess.run(bashCommand7, check=True, text=True, shell=True)
177
+
178
+ # Delete tmp files
179
+ bashCommand8 = os.path.join(f"rm {save_path}", f"tmp.pdb {save_path}", f"tmp1.pdb {save_path}", "tmp2.pdb")
180
+ subprocess.run(bashCommand8, check=True, text=True, shell=True)
181
+
182
+
183
+ ## In-house scripts
184
+
185
+ # Add SEQREF record to pdb file
186
+
187
+ def get_res_from_chain(pdb_path):
188
+ """
189
+ Get sequense of amino acid residues from the structure chain.
190
+ """
191
+
192
+ # Load structure
193
+ parser = PDBParser()
194
+ structure = parser.get_structure("ID", pdb_path)
195
+
196
+ # Get seq from chain
197
+ residues = []
198
+ chain = structure[0]["A"]
199
+ for residue in chain.get_residues():
200
+ residues.append(residue.resname)
201
+
202
+ return residues
203
+
204
+
205
+ def get_pdb_seqres_records(lst_res):
206
+ """
207
+ Construct the fixed-width records of a pdb file.
208
+ """
209
+
210
+ records = []
211
+ num_residues = len(lst_res)
212
+ record_counter = 0
213
+ while record_counter * 13 < num_residues:
214
+ start_idx = record_counter * 13
215
+ end_idx = min(start_idx + 13, num_residues)
216
+ residue_subset = lst_res[start_idx:end_idx]
217
+ record = 'SEQRES {:>3} {} {:>4} {:52}\n'.format(record_counter+1, "A", num_residues, ' '.join(residue_subset))
218
+ records.append(record)
219
+ record_counter += 1
220
+
221
+ return records
222
+
223
+
224
+ def add_refseq_record_to_pdb(path_structure):
225
+ """
226
+ Add the SEQREF records to the pdb file.
227
+ """
228
+
229
+ # Open the PDB file and get SEQRES insert index
230
+ with open(path_structure, 'r') as file:
231
+ pdb_lines = file.readlines()
232
+ insert_index = next(i for i, line in enumerate(pdb_lines) if line.startswith('MODEL'))
233
+
234
+ # Get seares records
235
+ residues = get_res_from_chain(path_structure)
236
+ seqres_records = get_pdb_seqres_records(residues)
237
+
238
+ # Insert the SEQRES records
239
+ pdb_lines[insert_index:insert_index] = seqres_records
240
+
241
+ # Save
242
+ with open(path_structure, 'w') as output_file:
243
+ output_file.truncate()
244
+ output_file.writelines(pdb_lines)
245
+
246
+
247
+ # Other functions
248
+
249
+ def get_list_fragmented_pdb(pdb_dir):
250
+ """
251
+ Given a directory including pdb files, return a list of tuples (Uniprot_ID, max AF_F).
252
+ """
253
+
254
+ # List pdb files
255
+ list_pdb = os.listdir(pdb_dir)
256
+ list_pdb = [file for file in list_pdb if not file.startswith("tmp") and file.endswith(".pdb") or file.endswith(".pdb.gz")]
257
+ list_pdb = [(file.split("-")[1], re.sub(r"\D", "", file.split("-")[2])) for file in list_pdb if file.split("-")[2][-1] != "M"]
258
+
259
+ # Get df with max fragment
260
+ df = pd.DataFrame(list_pdb, columns=["Uniprot_ID", "F"])
261
+ df["F"] = pd.to_numeric(df["F"])
262
+ df = df.groupby("Uniprot_ID").max()
263
+
264
+ # Get fragmented structures as list of (Uniprot_ID AF_F) tuples
265
+ df = df[df["F"] > 1].reset_index()
266
+
267
+ return list(df.to_records(index=False))
268
+
269
+
270
+ def save_unprocessed_ids(uni_ids, filename):
271
+
272
+ with open(filename, 'a') as file:
273
+ for id in uni_ids:
274
+ file.write(id + '\n')
275
+
276
+
277
+ # Wrapper function
278
+
279
+ def merge_af_fragments(input_dir, output_dir=None, af_version=4, gzip=False):
280
+ """
281
+ Run and parse DEGRONOPEDIA script to merge any AlphaFold fragments into a unique structure.
282
+
283
+ DEGRONOPEDIA - a web server for proteome-wide inspection of degrons
284
+ doi: 10.1101/2022.05.19.492622
285
+ https://degronopedia.com/degronopedia/about
286
+ """
287
+
288
+ if output_dir is None:
289
+ output_dir = input_dir
290
+ if gzip:
291
+ zip_ext = ".gz"
292
+ else:
293
+ zip_ext = ""
294
+
295
+ fragments = get_list_fragmented_pdb(input_dir)
296
+ if len(fragments) > 0:
297
+
298
+ # Create dir where to move original fragmented structures
299
+ path_original_frag = os.path.join(output_dir, "fragmented_pdbs")
300
+ if not os.path.exists(path_original_frag):
301
+ os.makedirs(path_original_frag)
302
+ checkpoint = os.path.join(path_original_frag, '.checkpoint.merge.txt')
303
+
304
+ if os.path.exists(checkpoint):
305
+ logger.debug("Merge fragments already performed: Skipping...")
306
+ else:
307
+ # Get list of fragmented Uniprot ID and max AF-F
308
+ not_processed = []
309
+ for uni_id, max_f in tqdm(fragments, total=len(fragments), desc="Merging AF fragments"):
310
+
311
+ processed = False
312
+
313
+ try:
314
+ degronopedia_af_merge(uni_id, input_dir, af_version, output_dir, gzip)
315
+ processed = True
316
+ except (PDBExceptions.PDBIOException, PDBExceptions.PDBConstructionException):
317
+ logger.warning(f"Could not process {uni_id} ({max_f} fragments)")
318
+ not_processed.append(uni_id)
319
+ f_path = os.path.join(output_dir, f"AF-{uni_id}-FM-model_v{af_version}.pdb")
320
+ if os.path.exists(f_path):
321
+ os.remove(f_path)
322
+ # Move the original fragmented structures
323
+ for f in range(1, max_f+1):
324
+ file = f"AF-{uni_id}-F{f}-model_v{af_version}.pdb{zip_ext}"
325
+ shutil.move(os.path.join(input_dir, file), path_original_frag)
326
+
327
+ # Rename merged structure and add refseq records to pdb
328
+ if processed:
329
+ tmp_name = os.path.join(output_dir, f"AF-{uni_id}-FM-model_v{af_version}.pdb")
330
+ name = os.path.join(output_dir, f"AF-{uni_id}-F{max_f}M-model_v{af_version}.pdb")
331
+ os.rename(tmp_name, name)
332
+ add_refseq_record_to_pdb(name)
333
+
334
+ if len(not_processed) > 0:
335
+ logger.warning(f"Not processed: {not_processed}")
336
+ with open(checkpoint, "w") as f:
337
+ f.write('')
338
+
339
+ save_unprocessed_ids(not_processed,
340
+ os.path.join(output_dir, "fragmented_pdbs", "ids_not_merged.txt"))
341
+ logger.info("Merge of structures completed!")
342
+
343
+ else:
344
+ logger.debug("Nothing to merge: Skipping...")
@@ -0,0 +1,125 @@
1
+ """
2
+ Module to generate datasets necessary to run Oncodrive3D.
3
+
4
+ The build is a pipeline that perform the following tasks:
5
+ - Download the PDB structures of the selected proteome
6
+ predicted by AlphaFold 2 from AlphaFold DB.
7
+ - Merge the overlapping structures processed as fragments.
8
+ - Extract AlphaFold model confidence (pLDDT).
9
+ - Generate a dataframe including Uniprot_ID, HUGO Symbol,
10
+ protein, DNA sequence, and other gene's information.
11
+ - Download AlphaFold predicted aligned error (PAE) from
12
+ AlphaFold DB and convert the files into npy format.
13
+ - Use the PDB structure and PAE to create maps of
14
+ probability of contacts (pCMAPs) for any protein of the
15
+ downloaded proteome with available PAE.
16
+ - Remove unnecessary temp files.
17
+ """
18
+
19
+
20
+ import os
21
+
22
+ import daiquiri
23
+
24
+ from scripts import __logger_name__
25
+ from scripts.datasets.af_merge import merge_af_fragments
26
+ from scripts.datasets.get_pae import get_pae
27
+ from scripts.datasets.get_structures import get_structures, mv_mane_pdb
28
+ from scripts.datasets.model_confidence import get_confidence
29
+ from scripts.datasets.parse_pae import parse_pae
30
+ from scripts.datasets.prob_contact_maps import get_prob_cmaps_mp
31
+ from scripts.datasets.seq_for_mut_prob import get_seq_df
32
+ from scripts.datasets.utils import get_species
33
+ from scripts.globals import clean_dir, clean_temp_files
34
+
35
+ logger = daiquiri.getLogger(__logger_name__ + ".build")
36
+
37
+ def build(output_datasets,
38
+ organism,
39
+ mane,
40
+ distance_threshold,
41
+ num_cores,
42
+ af_version,
43
+ mane_version):
44
+ """
45
+ Build datasets necessary to run Oncodrive3D.
46
+ """
47
+
48
+ # Empty directory
49
+ clean_dir(output_datasets, 'd', txt_file=True)
50
+
51
+ # Download PDB structures
52
+ species = get_species(organism)
53
+ logger.info("Downloading AlphaFold (AF) predicted structures...")
54
+ get_structures(path=os.path.join(output_datasets,"pdb_structures"),
55
+ species=species,
56
+ af_version=str(af_version),
57
+ threads=num_cores)
58
+ logger.info("Download of structures completed!")
59
+
60
+ # Merge fragmented structures
61
+ logger.info("Merging fragmented structures...")
62
+ merge_af_fragments(input_dir=os.path.join(output_datasets,"pdb_structures"),
63
+ gzip=True)
64
+
65
+ # Download PDB MANE structures
66
+ if species == "Homo sapiens" and mane == True:
67
+ logger.info("Downloading AlphaFold (AF) predicted structures overlap with MANE...")
68
+ get_structures(path=os.path.join(output_datasets,"pdb_structures_mane"),
69
+ species=species,
70
+ mane=True,
71
+ threads=num_cores)
72
+ mv_mane_pdb(output_datasets, "pdb_structures", "pdb_structures_mane")
73
+ logger.info("Download of MANE structures completed!")
74
+
75
+ # Create df including genes and proteins sequences & Hugo to Uniprot_ID mapping
76
+ logger.info("Generating dataframe for genes and proteins sequences...")
77
+ seq_df = get_seq_df(datasets_dir=output_datasets,
78
+ output_seq_df=os.path.join(output_datasets, "seq_for_mut_prob.tsv"),
79
+ organism=species,
80
+ mane=mane,
81
+ num_cores=num_cores,
82
+ mane_version=mane_version)
83
+ logger.info("Generation of sequences dataframe completed!")
84
+
85
+ # Get model confidence
86
+ logger.info("Extracting AF model confidence...")
87
+ get_confidence(input=os.path.join(output_datasets, "pdb_structures"),
88
+ output_dir=os.path.join(output_datasets),
89
+ seq_df=seq_df)
90
+
91
+ # Get PAE
92
+ logger.info("Downloading AF predicted aligned error (PAE)...")
93
+ get_pae(input_dir=os.path.join(output_datasets,"pdb_structures"),
94
+ output_dir=os.path.join(output_datasets,"pae"),
95
+ num_cores=num_cores,
96
+ af_version=str(af_version))
97
+
98
+ # Parse PAE
99
+ logger.info("Parsing PAE...")
100
+ parse_pae(input=os.path.join(output_datasets, 'pae'))
101
+ logger.info("Parsing PAE completed!")
102
+
103
+ # Get pCAMPs
104
+ logger.info("Generating contact probability maps (pCMAPs)..")
105
+ get_prob_cmaps_mp(input_pdb=os.path.join(output_datasets, "pdb_structures"),
106
+ input_pae=os.path.join(output_datasets, "pae"),
107
+ output=os.path.join(output_datasets,"prob_cmaps"),
108
+ distance=distance_threshold,
109
+ num_cores=num_cores)
110
+ logger.info("Generation pCMAPs completed!")
111
+
112
+ # Clean datasets
113
+ logger.info("Cleaning datasets...")
114
+ clean_temp_files(path=output_datasets)
115
+ logger.info("Datasets cleaning completed!")
116
+ logger.info("Datasets have been successfully built and are ready for analysis!")
117
+
118
+ if __name__ == "__main__":
119
+ build(output_datasets="/workspace/nobackup/scratch/oncodrive3d/datasets_mane",
120
+ organism="Homo sapiens",
121
+ mane=True,
122
+ distance_threshold=10,
123
+ num_cores=8,
124
+ af_version=4,
125
+ mane_version=1.3)
@@ -0,0 +1,78 @@
1
+ import os
2
+
3
+ import daiquiri
4
+ import requests
5
+ import time
6
+ import concurrent.futures
7
+ from tqdm import tqdm
8
+
9
+ from scripts import __logger_name__
10
+
11
+ logger = daiquiri.getLogger(__logger_name__ + ".build.PAE")
12
+
13
+
14
+ def download_pae(uniprot_id: str, af_version: int, output_dir: str) -> None:
15
+ """
16
+ Download Predicted Aligned Error (PAE) file from AlphaFold DB.
17
+
18
+ Args:
19
+ uniprot_id (str): Uniprot ID of the structure.
20
+ af_version (int): AlphaFold 2 version.
21
+ output_dir (str): Output directory where to download the PAE files.
22
+ """
23
+
24
+ file_path = os.path.join(output_dir, f"{uniprot_id}-F1-predicted_aligned_error.json")
25
+ download_url = f"https://alphafold.ebi.ac.uk/files/AF-{uniprot_id}-F1-predicted_aligned_error_v{af_version}.json"
26
+
27
+ i = 0
28
+ status = "INIT"
29
+ while status != "FINISHED":
30
+ i += 1
31
+ if status != "INIT":
32
+ time.sleep(30)
33
+ try:
34
+ response = requests.get(download_url, timeout=30)
35
+ content = response.content
36
+ if content.endswith(b'}]') and not content.endswith(b'</Error>'):
37
+ with open(file_path, 'wb') as output_file:
38
+ output_file.write(content)
39
+ status = "FINISHED"
40
+ except requests.exceptions.RequestException as e:
41
+ status = "ERROR"
42
+ if i % 10 == 0:
43
+ logger.debug(f"Request failed {e}: Retrying")
44
+
45
+
46
+ def get_pae(input_dir: str, output_dir: str, num_cores: int, af_version: int = 4) -> None:
47
+ """
48
+ Download Predicted Aligned Error (PAE) files for all non-fragmented PDB
49
+ structures in the input directory.
50
+
51
+ Args:
52
+ input_dir (str): Input directory including the PDB structures.
53
+ output_dir (str): Output directory where to download the PAE files.
54
+ num_cores (int): Number of cores for multithreading download.
55
+ af_version (int): AlphaFold 2 version (default is 4).
56
+ """
57
+
58
+ if not os.path.exists(output_dir):
59
+ os.makedirs(output_dir)
60
+
61
+ checkpoint = os.path.join(output_dir, '.checkpoint.txt')
62
+ if os.path.exists(checkpoint):
63
+ logger.debug("PAE already downloaded: Skipping...")
64
+ return
65
+
66
+ pdb_files = [file for file in os.listdir(input_dir) if file.startswith("AF-") and file.endswith(f"-model_v{af_version}.pdb.gz")]
67
+ uniprot_ids = [pdb_file.split("-")[1] for pdb_file in pdb_files]
68
+
69
+ with concurrent.futures.ThreadPoolExecutor(max_workers=num_cores) as executor:
70
+ tasks = [executor.submit(download_pae, uniprot_id, af_version, output_dir) for uniprot_id in uniprot_ids]
71
+
72
+ for _ in tqdm(concurrent.futures.as_completed(tasks), total=len(tasks), desc="Downloading PAE"):
73
+ pass
74
+
75
+ with open(checkpoint, "w") as f:
76
+ f.write('')
77
+
78
+ logger.info('Download of PAE completed!')
@@ -0,0 +1,107 @@
1
+ import logging
2
+ import os
3
+
4
+ import daiquiri
5
+ import time
6
+ import subprocess
7
+ import shutil
8
+
9
+ from scripts import __logger_name__
10
+ from scripts.datasets.utils import calculate_hash, download_single_file, extract_tar_file, assert_proteome_integrity, CHECKSUM
11
+
12
+ logger = daiquiri.getLogger(__logger_name__ + ".build.AF-pdb")
13
+
14
+ logging.getLogger('urllib3.connectionpool').setLevel(logging.WARNING)
15
+
16
+
17
+ def mv_mane_pdb(path_datasets, pdb_dir, mane_pdb_dir) -> None:
18
+ """
19
+ Move AF structures with overlap with MANE Select
20
+ transcripts to directory with AF structures from
21
+ human proteome. Overwrite any overlapping AF ID.
22
+ """
23
+
24
+ path_pdb = os.path.join(path_datasets, pdb_dir)
25
+ path_mane_pdb = os.path.join(path_datasets, mane_pdb_dir)
26
+ if not os.path.exists(path_mane_pdb):
27
+ os.makedirs(path_mane_pdb)
28
+
29
+ # Move MANE structures
30
+ for filename in [file for file in os.listdir(path_mane_pdb) if file.endswith(".pdb.gz") or file.endswith(".pdb")]:
31
+ pdb = os.path.join(path_pdb, filename)
32
+ pdb_mane = os.path.join(path_mane_pdb, filename)
33
+ shutil.move(pdb_mane, pdb)
34
+
35
+ # Move MANE metadata files
36
+ for filename in [file for file in os.listdir(os.path.join(path_mane_pdb)) if file.endswith(".csv") or file.endswith("readme.txt")]:
37
+ source_file = os.path.join(path_mane_pdb, filename)
38
+ dest_file = os.path.join(path_datasets, f"mane_{filename}")
39
+ shutil.move(source_file, dest_file)
40
+
41
+
42
+ def get_structures(path: str,
43
+ species: str = 'Homo sapiens',
44
+ mane: bool = False,
45
+ af_version: str = '4',
46
+ threads: int = 1,
47
+ max_attempts: int = 30) -> None:
48
+ """
49
+ Downloads AlphaFold predicted structures for a given organism and version.
50
+
51
+ Args:
52
+ path (str): Path where to save PDB structures.
53
+ species (str, optional): Species (human (default) or mouse). Defaults to 'human'.
54
+ af_version (str, optional): AlphaFold 2 version (4 as default). Defaults to '4'.
55
+ verbose (str, optional): Verbose (True (default) or False). Defaults to 'False'.
56
+
57
+ Example:
58
+ get_structures('datasets/pdb_structures', species='human', verbose='True')
59
+ """
60
+
61
+ logger.info(f"Selected species: {species}")
62
+
63
+ if not os.path.isdir(path):
64
+ os.makedirs(path)
65
+ logger.debug(f'mkdir {path}')
66
+
67
+ # Select proteome
68
+ if mane:
69
+ if species == "Homo sapiens":
70
+ proteome = f"mane_overlap_v{af_version}"
71
+ else:
72
+ raise RuntimeError(f"Structures with MANE transcripts overlap are available only for 'Homo sapiens'. Exiting...")
73
+ else:
74
+ if species == "Homo sapiens":
75
+ proteome = f"UP000005640_9606_HUMAN_v{af_version}"
76
+ elif species == "Mus musculus":
77
+ proteome = f"UP000000589_10090_MOUSE_v{af_version}"
78
+ else:
79
+ raise RuntimeError(f"Failed to recognize '{species}' as organism. Currently accepted ones are 'Homo sapiens' and 'Mus musculus'. Exiting...")
80
+
81
+ logger.debug(f"Proteome to download: {proteome}")
82
+ af_url = f"https://ftp.ebi.ac.uk/pub/databases/alphafold/latest/{proteome}.tar"
83
+ file_path = os.path.join(path, f"{proteome}.tar")
84
+
85
+ try:
86
+ ## STEP1 --- Download file
87
+ attempts = 0
88
+ status = "INIT"
89
+ while status != "PASS":
90
+ download_single_file(af_url, file_path, threads, proteome)
91
+ status = assert_proteome_integrity(file_path, proteome)
92
+ attempts += 1
93
+ if attempts >= max_attempts:
94
+ raise RuntimeError(f"Failed to download with integrity after {max_attempts} attempts. Exiting...")
95
+ time.sleep(10)
96
+
97
+ ## STEP2 --- Extract structures
98
+ logger.info(f'Extracting {file_path}')
99
+ extract_tar_file(file_path, path)
100
+
101
+ logger.info('Download structure: SUCCESS')
102
+ logger.debug(f"Structures downloaded in directory {path}")
103
+
104
+ except Exception as e:
105
+ logger.error('Download structure: FAIL')
106
+ logger.error(f"Error while downloading structures: {e}")
107
+ raise e