Fast-HInt-ppi 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- Fast_HInt/File_proteins.py +780 -0
- Fast_HInt/HInt.py +244 -0
- Fast_HInt/Scoring_HInt.py +954 -0
- Fast_HInt/Utils_HInt.py +1313 -0
- Fast_HInt/__init__.py +1 -0
- Fast_HInt/get_good_inter_pae.py +340 -0
- fast_hint_ppi-0.1.0.dist-info/METADATA +33 -0
- fast_hint_ppi-0.1.0.dist-info/RECORD +11 -0
- fast_hint_ppi-0.1.0.dist-info/WHEEL +5 -0
- fast_hint_ppi-0.1.0.dist-info/entry_points.txt +2 -0
- fast_hint_ppi-0.1.0.dist-info/top_level.txt +1 -0
Fast_HInt/Utils_HInt.py
ADDED
|
@@ -0,0 +1,1313 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import pickle
|
|
3
|
+
import numpy as np
|
|
4
|
+
import matplotlib.pyplot as plt
|
|
5
|
+
import csv
|
|
6
|
+
import logging
|
|
7
|
+
import subprocess
|
|
8
|
+
import multiprocessing
|
|
9
|
+
import pynvml
|
|
10
|
+
import copy
|
|
11
|
+
import signal
|
|
12
|
+
import glob
|
|
13
|
+
import math
|
|
14
|
+
import random
|
|
15
|
+
import json
|
|
16
|
+
import string
|
|
17
|
+
import time
|
|
18
|
+
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from Bio import SeqIO
|
|
21
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
22
|
+
from .Scoring_HInt import Score_interaction
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# Configure global logger
|
|
26
|
+
logging.basicConfig(
|
|
27
|
+
filename="./log_file/HInt.log", # Log file name
|
|
28
|
+
level=logging.INFO, # Log level
|
|
29
|
+
format="%(asctime)s - %(levelname)s - %(message)s" # Log format
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
# Use the logger configuration from HInt.py
|
|
33
|
+
logger = logging.getLogger()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def Define_informations() :
|
|
38
|
+
"""
|
|
39
|
+
Parse the HInt.txt configuration file and collect all user-defined parameters into a single dictionary.
|
|
40
|
+
|
|
41
|
+
- Reads key : value pairs from HInt.txt (comments starting with # are ignored)
|
|
42
|
+
- Verifies the presence of mandatory parameters
|
|
43
|
+
- Assigns default values to optional or empty parameters
|
|
44
|
+
- Normalizes paths (removes trailing '/')
|
|
45
|
+
- Parses interaction definitions (including multimers and interaction regions)
|
|
46
|
+
- Validates biological constraints (DeepLoc, homo-oligomer, number of baits)
|
|
47
|
+
|
|
48
|
+
Returns:
|
|
49
|
+
-------
|
|
50
|
+
Informations_dict : dict
|
|
51
|
+
Dictionary containing all validated and formatted configuration parameters required by HInt.
|
|
52
|
+
|
|
53
|
+
"""
|
|
54
|
+
logger.info("Defining informations")
|
|
55
|
+
Informations_dict = dict()
|
|
56
|
+
list_inf = ["Signal_peptide", "Homo-oligomer", "Interact_with", "Organism", "DeepLoc", "Regions", "Multimer_bait", "AlphaFold", "Max_protein_lenght", "Min_protein_lenght", "Path_Uniprot_ID", "Path_AlphaFold_Data", "Path_Pickle_Feature", "Path_Singularity_Image", "Path_MMseqs2_Data"]
|
|
57
|
+
with open("HInt.txt", "r") as file :
|
|
58
|
+
for lines in file :
|
|
59
|
+
if ":" in lines :
|
|
60
|
+
info = lines.split("#")[0]
|
|
61
|
+
informations_name = info.split(":")[0].strip().strip("\n")
|
|
62
|
+
informations = info.split(":")[1].strip().strip("\n")
|
|
63
|
+
Informations_dict[informations_name] = informations
|
|
64
|
+
for info in list_inf :
|
|
65
|
+
if info not in Informations_dict.keys() : #if settings file is not authentic
|
|
66
|
+
if info in ["Interact_with","Path_Uniprot_ID", "Path_AlphaFold_Data", "Path_Pickle_Feature"] :
|
|
67
|
+
raise ValueError(f"HInt.txt file is compromised, verify the file. {info} is missing")
|
|
68
|
+
elif info in ["Signal_peptide","Homo-oligomer","Path_MMseqs2_Data","Regions","Multimer_bait","DeepLoc","AlphaFold","Max_protein_lenght","Min_protein_lenght","Organism"] :
|
|
69
|
+
Informations_dict[info] = ""
|
|
70
|
+
|
|
71
|
+
### Normalize all configuration values
|
|
72
|
+
for informations_key in Informations_dict.keys() : #verify all informations and set default value
|
|
73
|
+
if type(Informations_dict[informations_key]) is str and Informations_dict[informations_key].endswith("/") : #avoid error in path
|
|
74
|
+
Informations_dict[informations_key] = Informations_dict[informations_key][:-1]
|
|
75
|
+
if len(Informations_dict[informations_key]) == 0 :
|
|
76
|
+
logger.info(f'{informations_key} is empty')
|
|
77
|
+
if informations_key == "Path_ccp4" :
|
|
78
|
+
if os.path.isdir("/opt/xtal/ccp4-9") == True :
|
|
79
|
+
logger.info("Set ccp4 path by default on /opt/xtal/ccp4-9")
|
|
80
|
+
Informations_dict[informations_key] = "/opt/xtal/ccp4-9"
|
|
81
|
+
else :
|
|
82
|
+
raise ValueError("Path_ccp4 is empty and ccp4 is not found in /opt/xtal/ccp4-9. You need to install ccp4 or set the path of ccp4 in HInt.txt")
|
|
83
|
+
elif informations_key == "Path_AlphaFold_Data" :
|
|
84
|
+
if os.path.isdir("./alphadata") == True :
|
|
85
|
+
logger.info("Set AlphaFold data by default on ./alphadata")
|
|
86
|
+
Informations_dict[informations_key] = "./alphadata"
|
|
87
|
+
else :
|
|
88
|
+
raise ValueError("Path_AlphaFold_Data is empty and AlphaFold data is not found in ./alphadata. You need to download AlphaFold data or set the path of AlphaFold data in HInt.txt")
|
|
89
|
+
elif informations_key == "Path_Pickle_Feature" :
|
|
90
|
+
logger.info("Set pickle feature path by default on ./feature")
|
|
91
|
+
Informations_dict[informations_key] = "./feature"
|
|
92
|
+
elif informations_key == "Path_MMseqs2_Data" :
|
|
93
|
+
logger.info("/!\ local MMseqs2 GPU will not be used")
|
|
94
|
+
elif informations_key == "Signal_peptide" :
|
|
95
|
+
Informations_dict[informations_key] = "None"
|
|
96
|
+
elif informations_key == "Homo-oligomer" :
|
|
97
|
+
Informations_dict[informations_key] = "1"
|
|
98
|
+
elif informations_key == "DeepLoc" :
|
|
99
|
+
Informations_dict[informations_key] = "None"
|
|
100
|
+
elif informations_key == "AlphaFold" :
|
|
101
|
+
Informations_dict[informations_key] = "2"
|
|
102
|
+
logger.info("Set AlphaFold version by default on AlphaFold2")
|
|
103
|
+
elif informations_key == "Min_protein_lenght" :
|
|
104
|
+
Informations_dict[informations_key] = "20"
|
|
105
|
+
logger.info("Minimum lenght for prey protein set by default 20 to AA")
|
|
106
|
+
elif informations_key == "Max_protein_lenght" :
|
|
107
|
+
Informations_dict[informations_key] = ""
|
|
108
|
+
elif informations_key == "Organism" :
|
|
109
|
+
Informations_dict[informations_key] = "None"
|
|
110
|
+
if len(Informations_dict[informations_key]) != 0 :
|
|
111
|
+
if informations_key == "Path_AlphaFold_Data" :
|
|
112
|
+
if os.path.isdir(Informations_dict[informations_key]) == False :
|
|
113
|
+
raise ValueError(f"Path set for AlphaFold database doesn't exist : {Informations_dict[informations_key]}")
|
|
114
|
+
elif informations_key == "Path_MMseqs2_Data" :
|
|
115
|
+
if os.path.isdir(Informations_dict[informations_key]) == False :
|
|
116
|
+
raise ValueError (f"Path set for MMseq database doesn't exist : {Informations_dict[informations_key]}")
|
|
117
|
+
elif informations_key == "Path_ccp4" :
|
|
118
|
+
if os.path.isdir(Informations_dict[informations_key]) == False :
|
|
119
|
+
raise ValueError (f"Path set for ccp4 doesn't exist : {Informations_dict[informations_key]}")
|
|
120
|
+
### Parse protein-protein interaction definitions
|
|
121
|
+
# Multiple baits
|
|
122
|
+
# Multimeric baits
|
|
123
|
+
# Region-specific interactions
|
|
124
|
+
if informations_key == "Interact_with" :
|
|
125
|
+
regions_dict = dict()
|
|
126
|
+
new_baits_list = list()
|
|
127
|
+
if "[" in Informations_dict["Interact_with"] : #create multimer for bait
|
|
128
|
+
list_complex = [prot.strip("[").strip(",").strip() for prot in Informations_dict["Interact_with"].split("]")]
|
|
129
|
+
if "" in list_complex :
|
|
130
|
+
list_complex.remove("")
|
|
131
|
+
nbr_baits = len(list_complex)
|
|
132
|
+
list_baits = [prot.strip("[").strip("]") for prot in Informations_dict["Interact_with"].split(",")]
|
|
133
|
+
Informations_dict["Multimer_bait"] = list_complex
|
|
134
|
+
else :
|
|
135
|
+
list_baits = [prot.strip(",").strip() for prot in Informations_dict["Interact_with"].split(",")]
|
|
136
|
+
nbr_baits = len(list_baits)
|
|
137
|
+
Informations_dict["Multimer_bait"] = list_baits
|
|
138
|
+
if nbr_baits > 3 :
|
|
139
|
+
raise ValueError("HInt don't support more than 3 differents baits")
|
|
140
|
+
for prot in list_baits :
|
|
141
|
+
if "-" in prot and "(" in prot and ")" in prot : #if region is specified for the bait :
|
|
142
|
+
name_prot = prot.split("(")[0]
|
|
143
|
+
new_baits_list.append(name_prot.strip())
|
|
144
|
+
regions_dict[name_prot.strip()] = prot.split("(")[1].strip(")")
|
|
145
|
+
for i,multimer in enumerate(Informations_dict["Multimer_bait"]) :
|
|
146
|
+
if prot in multimer :
|
|
147
|
+
new_multimer = multimer.replace(prot,name_prot.strip())
|
|
148
|
+
Informations_dict["Multimer_bait"][i] = new_multimer
|
|
149
|
+
else :
|
|
150
|
+
if prot.strip() in regions_dict.keys() and regions_dict[prot.strip()] != "0-0" :
|
|
151
|
+
raise ValueError(f"HInt don't support multiple regions for protein bait : {prot.strip()}")
|
|
152
|
+
regions_dict[prot.strip()] = "0-0"
|
|
153
|
+
new_baits_list.append(prot.strip())
|
|
154
|
+
Informations_dict["Regions"] = regions_dict
|
|
155
|
+
Informations_dict["Interact_with"] = new_baits_list
|
|
156
|
+
|
|
157
|
+
### Homo-oligomer check
|
|
158
|
+
if informations_key == "Homo-oligomer" :
|
|
159
|
+
if int(Informations_dict[informations_key]) == False :
|
|
160
|
+
raise ValueError(f"Homo-oligomer is not an integer")
|
|
161
|
+
if Informations_dict[informations_key] == "0" :
|
|
162
|
+
Informations_dict[informations_key] = "1"
|
|
163
|
+
### Organism check
|
|
164
|
+
if informations_key == "Organism" :
|
|
165
|
+
if Informations_dict[informations_key] not in ["arch", "gram+", "gram-", "euk", "None"] :
|
|
166
|
+
raise ValueError(f"Incorrect Organism value : {Informations_dict[informations_key]}")
|
|
167
|
+
|
|
168
|
+
### DeepLoc check
|
|
169
|
+
if informations_key == "DeepLoc" :
|
|
170
|
+
if Informations_dict["Organism"] == "euk" : #euk
|
|
171
|
+
for value in Informations_dict[informations_key].split(","):
|
|
172
|
+
if value.strip() not in ["Cytoplasm", "Nucleus", "Extracellular", "Cell membrane", "Mitochondrion", "Plastid", "Endoplasmic reticulum", "Lysosome/Vacuole", "Golgo apparatus", "Peroxisome", "None"] :
|
|
173
|
+
raise ValueError(f"Incorrect DeepLoc value : {value}")
|
|
174
|
+
else : #other
|
|
175
|
+
for value in Informations_dict[informations_key].split(","):
|
|
176
|
+
if value.strip() not in ["Cell wall & surface", "Extracellular", "Cytoplasmic", "Cytoplasmic Membrane", "Outer Membrane", "Periplasmic", "None"] :
|
|
177
|
+
raise ValueError(f"Incorrect DeepLocPro value : {value}")
|
|
178
|
+
|
|
179
|
+
if len(Informations_dict["Signal_peptide"]) == 0 and len(Informations_dict["DeepLoc"]) == 0 and len(Informations_dict["Homo-oligomer"]) == 0 and len(Informations_dict["Interact_with"]) == 0 : #no info
|
|
180
|
+
logger.info("Need information to discriminate the potential homolog/interolog")
|
|
181
|
+
exit()
|
|
182
|
+
return(Informations_dict)
|
|
183
|
+
|
|
184
|
+
def run_deeploc(file, org, need_DeepLoc, GPU) :
|
|
185
|
+
"""
|
|
186
|
+
Run DeepLoc (DeepLoc2 or DeepLocPro) to predict the subcellular localization of proteins requiring DeepLoc annotation.
|
|
187
|
+
|
|
188
|
+
- Extracts protein sequences (with signal peptide handling) from a File_proteins object
|
|
189
|
+
- Writes a temporary FASTA file containing only proteins needing DeepLoc prediction
|
|
190
|
+
- Runs the appropriate DeepLoc software depending on the organism type
|
|
191
|
+
- Parses the output CSV file
|
|
192
|
+
- Stores predicted localizations in the result dictionary
|
|
193
|
+
|
|
194
|
+
Parameters :
|
|
195
|
+
----------
|
|
196
|
+
file : object of class File_proteins
|
|
197
|
+
org : str
|
|
198
|
+
need_DeepLoc : list
|
|
199
|
+
GPU : list
|
|
200
|
+
|
|
201
|
+
"""
|
|
202
|
+
result_dict = file.get_result_dict()
|
|
203
|
+
prot_seq = file.get_proteins_sequence_SP()
|
|
204
|
+
deeploc = file.get_deeploc()
|
|
205
|
+
GPU_str = ""
|
|
206
|
+
for nbr_GPU in GPU :
|
|
207
|
+
GPU_str += nbr_GPU + ","
|
|
208
|
+
GPU_str = GPU_str.strip(",")
|
|
209
|
+
file_name = file.get_file_name().split("/")[-1]
|
|
210
|
+
ext_f = "." + file_name.split(".")[-1]
|
|
211
|
+
fasta_file = file_name.replace(ext_f,"_msa.fasta")
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
if os.path.exists("log_file/result_deeploc") == True :
|
|
215
|
+
os.system("rm -r log_file/result_deeploc")
|
|
216
|
+
dp_lines = str()
|
|
217
|
+
for protein in need_DeepLoc :
|
|
218
|
+
dp_lines += ">"+protein+"\n"+prot_seq[protein]+"\n"
|
|
219
|
+
with open(f"log_file/{fasta_file}", "w") as dp_file :
|
|
220
|
+
dp_file.write(dp_lines)
|
|
221
|
+
|
|
222
|
+
if org == "euk" :
|
|
223
|
+
logger.info("Start DeepLoc eucaryote")
|
|
224
|
+
software = "deeploc2"
|
|
225
|
+
cmd = f"CUDA_VISIBLE_DEVICES={GPU_str} {software} -f log_file/{fasta_file} -o log_file/result_deeploc -d cuda"
|
|
226
|
+
else :
|
|
227
|
+
logger.info("Start DeepLocPro")
|
|
228
|
+
software = "deeplocpro"
|
|
229
|
+
if org == "gram-" :
|
|
230
|
+
group = "negative"
|
|
231
|
+
if org == "gram+" :
|
|
232
|
+
group = "positive"
|
|
233
|
+
if org == "arch" :
|
|
234
|
+
group = "archaea"
|
|
235
|
+
cmd = f"CUDA_VISIBLE_DEVICES={GPU_str} {software} -f log_file/{fasta_file} -o log_file/result_deeploc -d cuda -g {group}" #deeplocpro don't use all GPU
|
|
236
|
+
os.system(cmd)
|
|
237
|
+
DeepLoc_file = os.listdir(f"log_file/result_deeploc")
|
|
238
|
+
if software == "deeplocpro" :
|
|
239
|
+
min_index = 3
|
|
240
|
+
min_score = 0.1
|
|
241
|
+
index_name = 1
|
|
242
|
+
else :
|
|
243
|
+
min_index = 4
|
|
244
|
+
min_score = 0.4
|
|
245
|
+
index_name = 0
|
|
246
|
+
with open(f"log_file/result_deeploc/{DeepLoc_file[0]}", "r") as DL_file :
|
|
247
|
+
reader = csv.reader(DL_file, delimiter=',')
|
|
248
|
+
for index, line in enumerate(reader) :
|
|
249
|
+
compartment = str()
|
|
250
|
+
if index == 0 :
|
|
251
|
+
first_line = line #save title name
|
|
252
|
+
else :
|
|
253
|
+
protein = line[index_name]
|
|
254
|
+
for index, score in enumerate(line) :
|
|
255
|
+
if index >= min_index and float(score) > min_score :
|
|
256
|
+
compartment += first_line[index] + "|"
|
|
257
|
+
compartment = compartment.strip("|")
|
|
258
|
+
|
|
259
|
+
deeploc[protein] = compartment
|
|
260
|
+
result_dict[protein]["DeepLoc"] = deeploc[protein]
|
|
261
|
+
file.set_result_dict(result_dict)
|
|
262
|
+
file.set_deeploc(deeploc)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def run_SP (file, Informations_dict, need_SP, need_msa) :
|
|
267
|
+
"""
|
|
268
|
+
Remove signal peptides from protein sequences using SignalP and generate new FASTA sequences without signal peptides.
|
|
269
|
+
|
|
270
|
+
- Runs SignalP on proteins requiring signal peptide (SP) detection
|
|
271
|
+
- Extracts cleavage positions from SignalP output
|
|
272
|
+
- Generates new sequences without signal peptides
|
|
273
|
+
- Updates internal sequence storage
|
|
274
|
+
- Returns the list of proteins still requiring MSA computation
|
|
275
|
+
- Updates the result dictionary with signal peptide presence/absence
|
|
276
|
+
|
|
277
|
+
If a protein already has an MSA but does not yet have a sequence without signal peptide, it will be processed here but not returned for MSA.
|
|
278
|
+
|
|
279
|
+
Parameters :
|
|
280
|
+
----------
|
|
281
|
+
file : object of class File_proteins
|
|
282
|
+
Informations_dict : dict
|
|
283
|
+
need_SP : list
|
|
284
|
+
List of protein identifiers requiring signal peptide detection/removal.
|
|
285
|
+
need_msa : list
|
|
286
|
+
List of protein identifiers requiring MSA generation.
|
|
287
|
+
|
|
288
|
+
Returns :
|
|
289
|
+
----------
|
|
290
|
+
new_need_msa : list
|
|
291
|
+
"""
|
|
292
|
+
final_file = str()
|
|
293
|
+
SP_signal = 0
|
|
294
|
+
file_name = file.get_file_name().split("/")[-1]
|
|
295
|
+
ext_f = "." + file_name.split(".")[-1]
|
|
296
|
+
fasta_file = file_name.replace(ext_f,"_msa.fasta")
|
|
297
|
+
output_file = fasta_file.replace(".fasta","")
|
|
298
|
+
new_fasta_dict = file.get_proteins_sequence_no_SP()
|
|
299
|
+
result_dict = file.get_result_dict()
|
|
300
|
+
if_prot_SP = file.get_prot_SP()
|
|
301
|
+
prot_SP_len = dict()
|
|
302
|
+
|
|
303
|
+
file.create_fasta_file(True, need_SP)
|
|
304
|
+
cmd = f"signalp -fasta log_file/{fasta_file} -org {Informations_dict['Organism']} -prefix log_file/{output_file}"
|
|
305
|
+
os.system(cmd)
|
|
306
|
+
|
|
307
|
+
file_signalp = fasta_file.replace(".fasta","_summary.signalp5")
|
|
308
|
+
with open(f"log_file/{file_signalp}","r") as fh :
|
|
309
|
+
for line in fh :
|
|
310
|
+
new_line = line.split("\t")
|
|
311
|
+
if new_line[1] != "OTHER" and new_line[0][0] != "#" :
|
|
312
|
+
prot_SP_len[new_line[0]] = new_line[len(new_line)-1].split("-")[1].split(".")[0]
|
|
313
|
+
result_dict[new_line[0]]["Signal_peptide"] = "Yes"
|
|
314
|
+
if_prot_SP[new_line[0]] = "Yes"
|
|
315
|
+
elif new_line[1] == "OTHER" and new_line[0][0] != "#" :
|
|
316
|
+
result_dict[new_line[0]]["Signal_peptide"] = "No"
|
|
317
|
+
if_prot_SP[new_line[0]] = "No"
|
|
318
|
+
with open(f"log_file/{fasta_file}", "r") as fa_file :
|
|
319
|
+
for line2 in fa_file :
|
|
320
|
+
new_line2 = line2
|
|
321
|
+
if SP_signal == 0 and line2[0] != ">" :
|
|
322
|
+
new_line2 = line2
|
|
323
|
+
new_fasta_dict[save_key] = line2.strip("\n")
|
|
324
|
+
if int(SP_signal) > 0 :
|
|
325
|
+
new_line2 = line2[int(SP_signal)-1:len(line2)]
|
|
326
|
+
new_fasta_dict[save_key] = line2[int(SP_signal)-1:len(line2)].strip("\n")
|
|
327
|
+
SP_signal = 0
|
|
328
|
+
if line2[0] == ">" :
|
|
329
|
+
save_key = line2[1:len(line2)-1]
|
|
330
|
+
if str(line2[1:len(line2)-1]) in prot_SP_len.keys() :
|
|
331
|
+
SP_signal = prot_SP_len[line2[1:len(line2)-1]]
|
|
332
|
+
final_file = final_file + new_line2
|
|
333
|
+
|
|
334
|
+
file.set_proteins_sequence_no_SP(new_fasta_dict)
|
|
335
|
+
file.set_result_dict(result_dict)
|
|
336
|
+
file.set_prot_SP(if_prot_SP)
|
|
337
|
+
return need_msa
|
|
338
|
+
|
|
339
|
+
def check_exist_MSA(file, Informations_dict, need_msa) :
|
|
340
|
+
"""
|
|
341
|
+
Check for the existence of precomputed MSAs.
|
|
342
|
+
- Searches for precomputed MSA
|
|
343
|
+
- Updates the list of proteins requiring MSA generation based on existing files
|
|
344
|
+
|
|
345
|
+
Parameters :
|
|
346
|
+
----------
|
|
347
|
+
file : object of class File_proteins
|
|
348
|
+
Informations_dict : dict
|
|
349
|
+
need_msa : list
|
|
350
|
+
|
|
351
|
+
Returns :
|
|
352
|
+
----------
|
|
353
|
+
new_need_msa : list
|
|
354
|
+
List of protein identifiers still requiring MSA generation after checking for existing files.
|
|
355
|
+
"""
|
|
356
|
+
new_fasta_dict = file.get_proteins_sequence_no_SP()
|
|
357
|
+
new_need_msa = list()
|
|
358
|
+
for prot in need_msa :
|
|
359
|
+
if os.path.isfile(f"{Informations_dict['Path_Pickle_Feature']}/{prot}.a3m") == False :
|
|
360
|
+
new_need_msa.append(prot)
|
|
361
|
+
if os.path.isfile(f"{Informations_dict['Path_Pickle_Feature']}/{prot}.a3m") == True : #if MSA already exist but have SP
|
|
362
|
+
with open(f"{Informations_dict['Path_Pickle_Feature']}/{prot}.a3m", "r") as msa_file :
|
|
363
|
+
b = "no"
|
|
364
|
+
for line in msa_file :
|
|
365
|
+
if b == "yes" :
|
|
366
|
+
first_line = line.strip("\n")
|
|
367
|
+
break
|
|
368
|
+
if line[0] == ">" :
|
|
369
|
+
b = "yes"
|
|
370
|
+
if first_line != new_fasta_dict[prot] : #if sequence with SP is different from sequence without SP, modify the MSA file with the new sequence without SP
|
|
371
|
+
cmd_rm = f"rm -r {Informations_dict['Path_Pickle_Feature']}/{prot}*"
|
|
372
|
+
os.system(cmd_rm)
|
|
373
|
+
new_need_msa.append(prot)
|
|
374
|
+
return new_need_msa
|
|
375
|
+
|
|
376
|
+
def create_feature (file, Informations_dict, GPU, CPU, need_msa, need_pkl) :
|
|
377
|
+
"""
|
|
378
|
+
Generate MSAs and AlphaFold feature pickle files for a set of proteins.
|
|
379
|
+
|
|
380
|
+
- Searches for precomputed MSAs in the AlphaFold database
|
|
381
|
+
- Generates MSAs using ColabFold/MMseqs2 when missing
|
|
382
|
+
- Uses CPU parallelization (mafft) and GPU acceleration (MMseqs2 GPU)
|
|
383
|
+
- Updates the lists of proteins requiring MSA or pickle generation
|
|
384
|
+
|
|
385
|
+
Parameters :
|
|
386
|
+
----------
|
|
387
|
+
file : object of class File_proteins
|
|
388
|
+
informations_dict : dict
|
|
389
|
+
GPU : list
|
|
390
|
+
CPU : int
|
|
391
|
+
need_msa : list
|
|
392
|
+
need_pkl : list
|
|
393
|
+
|
|
394
|
+
"""
|
|
395
|
+
if need_msa == [] and need_pkl == [] :
|
|
396
|
+
return
|
|
397
|
+
AF_version = Informations_dict["AlphaFold"]
|
|
398
|
+
Path_AlphaFold_Data = Informations_dict["Path_AlphaFold_Data"]
|
|
399
|
+
Path_Pickle_Feature = Informations_dict["Path_Pickle_Feature"]
|
|
400
|
+
uniprot_prot = file.get_uniprot_prot()
|
|
401
|
+
prot_no_SP = file.get_proteins_sequence_no_SP()
|
|
402
|
+
prot_SP = file.get_proteins_sequence_SP()
|
|
403
|
+
file_name = file.get_file_name()
|
|
404
|
+
ext_f = "." + file_name.split(".")[-1]
|
|
405
|
+
msa_name = file_name.replace(ext_f,"_msa.fasta")
|
|
406
|
+
retry_queue = list()
|
|
407
|
+
GPU_str = str()
|
|
408
|
+
for nbr_GPU in GPU :
|
|
409
|
+
GPU_str += nbr_GPU + ","
|
|
410
|
+
GPU_str = GPU_str.strip(",")
|
|
411
|
+
#logger.info(f"GPU use : {GPU_str}")
|
|
412
|
+
logger.info(f"{len(need_msa)} proteins need MSA")
|
|
413
|
+
logger.info(f"{len(need_pkl)} proteins need pkl files")
|
|
414
|
+
if os.path.exists(Path_Pickle_Feature) == False :
|
|
415
|
+
os.system(f"mkdir {Path_Pickle_Feature}")
|
|
416
|
+
|
|
417
|
+
def AFDB_msa_search(Path_Pickle_Feature,prot_SP,prot_no_SP,generated_msa) :
|
|
418
|
+
retry_queue = list() #list of protein to retry if error during MSA search in AFdb (too many request on server)
|
|
419
|
+
with ThreadPoolExecutor(max_workers=max_workers_mafft) as executor :
|
|
420
|
+
for protein in generated_msa : #check if prot have an MSA in alphafold database
|
|
421
|
+
l_p = len(protein)
|
|
422
|
+
if l_p >= 5 and l_p <= 10 and "_" not in protein and protein in uniprot_prot : #if not, is not an UniprotID #avoid name with UniprotID but different sequence
|
|
423
|
+
on_afdb = True
|
|
424
|
+
time.sleep(0.2) #avoid too many request on AFdb server
|
|
425
|
+
url = f"https://alphafold.ebi.ac.uk/files/msa/AF-{protein}-F1-msa_v6.a3m" #do it linearly because of error with parallelization, due to too many request on AFdb server
|
|
426
|
+
msa_in = f"{Path_Pickle_Feature}/{protein}.a3m"
|
|
427
|
+
result = subprocess.run(["wget", "-O", msa_in, url], stderr=subprocess.PIPE)
|
|
428
|
+
if result.returncode != 0 :
|
|
429
|
+
err = result.stderr.decode()
|
|
430
|
+
on_afdb = False
|
|
431
|
+
if "404" in err : #404 error mean that MSA is not available in AFdb
|
|
432
|
+
os.remove(msa_in)
|
|
433
|
+
if "429" in err or "Too Many Requests" in err :
|
|
434
|
+
retry_queue.append(protein)
|
|
435
|
+
|
|
436
|
+
else :
|
|
437
|
+
nbr_line = sum(1 for _ in open(msa_in))
|
|
438
|
+
if nbr_line < 3 : #if MSA have only 1 sequence, not useful for AF2 prediction, so generated it with mmseqs2
|
|
439
|
+
on_afdb = False
|
|
440
|
+
os.remove(msa_in)
|
|
441
|
+
if on_afdb == True :
|
|
442
|
+
SP = len(prot_SP[protein]) - len(prot_no_SP[protein])
|
|
443
|
+
if SP > 0 : #if no SP don't modify the MSA
|
|
444
|
+
executor.submit(fetch_trim_mafft, protein, Path_Pickle_Feature, prot_SP, prot_no_SP)
|
|
445
|
+
logger.info(f"MSA for {protein} processed")
|
|
446
|
+
need_msa.remove(protein) #msa found
|
|
447
|
+
need_pkl.append(protein)
|
|
448
|
+
return retry_queue
|
|
449
|
+
|
|
450
|
+
start = time.time()
|
|
451
|
+
generated_msa = copy.deepcopy(need_msa)
|
|
452
|
+
if need_msa != [] :
|
|
453
|
+
#ThreadPool to parallelise Mafft
|
|
454
|
+
max_workers_mafft = CPU # how many jobs in parallel
|
|
455
|
+
|
|
456
|
+
#Look for MSA in AlphaFold database
|
|
457
|
+
logger.info(f"Search MSA in AlphaFold database")
|
|
458
|
+
|
|
459
|
+
retry_queue = AFDB_msa_search(Path_Pickle_Feature,prot_SP,prot_no_SP,generated_msa)
|
|
460
|
+
|
|
461
|
+
while retry_queue != [] : #retry if error during MSA search in AFdb
|
|
462
|
+
retry_queue = AFDB_msa_search(Path_Pickle_Feature,prot_SP,prot_no_SP,retry_queue)
|
|
463
|
+
|
|
464
|
+
logger.info("MSA search in AlphaFold database complete")
|
|
465
|
+
file.create_fasta_file(False, need_msa, need_pkl)
|
|
466
|
+
|
|
467
|
+
#Create MSA files with ColabFold mmseq2 classic pipeline for proteins without MSA, and pickle files for proteins generated with MMseqs2 GPU
|
|
468
|
+
if AF_version == "2" : # Cause AF3 if too fast for hmmer MSA generation, so we use mmseqs2 classic pipeline for AF3
|
|
469
|
+
mmseqs2 = "True"
|
|
470
|
+
else :
|
|
471
|
+
mmseqs2 = "False"
|
|
472
|
+
if len(need_msa) > 0 :
|
|
473
|
+
max_workers_feature = CPU
|
|
474
|
+
futures_list = []
|
|
475
|
+
with ThreadPoolExecutor(max_workers=max_workers_feature) as executor : #CPU parallelization
|
|
476
|
+
for protein in need_msa :
|
|
477
|
+
msa_name = f"./log_file/{protein}_msa.fasta"
|
|
478
|
+
cmd = ["create_individual_features.py",
|
|
479
|
+
f"--fasta_paths={msa_name}",
|
|
480
|
+
f"--data_dir=/data/alphadata_v2",
|
|
481
|
+
"--save_msa_files=True",
|
|
482
|
+
f"--output_dir={Path_Pickle_Feature}",
|
|
483
|
+
"--max_template_date=2024-05-02",
|
|
484
|
+
"--skip_existing=True",
|
|
485
|
+
f"--use_mmseqs2={mmseqs2}",
|
|
486
|
+
"--use_precomputed_msas=True"]
|
|
487
|
+
futures_list.append(executor.submit(create_ind_feature, protein, msa_name, prot_no_SP, cmd))
|
|
488
|
+
end = time.time()
|
|
489
|
+
elapsed = end - start
|
|
490
|
+
logger.info("Create MSA take "+ str(elapsed/60)+" minutes")
|
|
491
|
+
|
|
492
|
+
#Create pkl files for proteins without pkl file (from MSA found in AFdb or error during MSA generation with ColabFold mmseqs2)
|
|
493
|
+
if len(need_pkl) > 0 :
|
|
494
|
+
max_workers_feature = CPU
|
|
495
|
+
futures_list = []
|
|
496
|
+
|
|
497
|
+
with ThreadPoolExecutor(max_workers=max_workers_feature) as executor : #CPU parallelization
|
|
498
|
+
for protein in need_pkl :
|
|
499
|
+
pkl_file = f"./log_file/{protein}_pkl.fasta"
|
|
500
|
+
cmd2 = ["create_individual_features.py",
|
|
501
|
+
f"--fasta_paths={pkl_file}",
|
|
502
|
+
f"--data_dir={Path_AlphaFold_Data}",
|
|
503
|
+
"--save_msa_files=True",
|
|
504
|
+
f"--output_dir={Path_Pickle_Feature}",
|
|
505
|
+
"--max_template_date=2024-05-02",
|
|
506
|
+
"--skip_existing=True",
|
|
507
|
+
"--use_mmseqs2=True",
|
|
508
|
+
"--use_precomputed_msas=True",
|
|
509
|
+
"--re_search_templates_mmseqs2=True"]
|
|
510
|
+
futures_list.append(executor.submit(create_ind_feature, protein, pkl_file, prot_no_SP, cmd2))
|
|
511
|
+
|
|
512
|
+
def create_ind_feature(protein, pkl_file, prot_no_SP, cmd) :
|
|
513
|
+
"""
|
|
514
|
+
Create a pickle file for a single protein using the create_individual_features.py script. Allow parallelization.
|
|
515
|
+
|
|
516
|
+
Parameters :
|
|
517
|
+
----------
|
|
518
|
+
protein : str
|
|
519
|
+
pkl_file : str
|
|
520
|
+
prot_no_SP : dict
|
|
521
|
+
cmd : str
|
|
522
|
+
"""
|
|
523
|
+
with open(pkl_file, "w") as pkl_f :
|
|
524
|
+
pkl_f.write(f">{protein}\n{prot_no_SP[protein]}\n")
|
|
525
|
+
process = subprocess.Popen(cmd, stderr=subprocess.STDOUT,stdout=subprocess.PIPE, text=True, bufsize=1, universal_newlines=True)
|
|
526
|
+
for line in process.stdout :
|
|
527
|
+
print(f"{line}", end="") # stream live
|
|
528
|
+
os.remove(pkl_file)
|
|
529
|
+
process.wait()
|
|
530
|
+
|
|
531
|
+
def fetch_trim_mafft(protein, Path_Pickle_Feature, prot_SP, prot_no_SP) :
|
|
532
|
+
"""
|
|
533
|
+
Trim signal peptides if present, and realign the MSA using MAFFT.
|
|
534
|
+
|
|
535
|
+
- Removes signal peptide residues from all sequences when necessary
|
|
536
|
+
- Filters out excessively short or invalid sequences after trimming
|
|
537
|
+
- Realigns the trimmed MSA using MAFFT
|
|
538
|
+
- Reformats the alignment back to A3M format
|
|
539
|
+
|
|
540
|
+
Parameters :
|
|
541
|
+
----------
|
|
542
|
+
protein : str
|
|
543
|
+
Path_Pickle_Feature : str
|
|
544
|
+
prot_SP : dict
|
|
545
|
+
prot_no_SP : dict
|
|
546
|
+
"""
|
|
547
|
+
msa_in = f"{Path_Pickle_Feature}/{protein}.a3m"
|
|
548
|
+
aln_out = f"{Path_Pickle_Feature}/{protein}.aln"
|
|
549
|
+
|
|
550
|
+
SP = len(prot_SP[protein]) - len(prot_no_SP[protein])
|
|
551
|
+
trimmed_records = []
|
|
552
|
+
for rec in SeqIO.parse(msa_in, "fasta") :
|
|
553
|
+
new_seq = rec.seq[SP:]
|
|
554
|
+
len_aa_count = sum(1 for c in new_seq if c.isupper())
|
|
555
|
+
if len_aa_count < 10 or not any(c.isupper() for c in str(new_seq)) : #skip short sequence after SP cut or sequence don't have aa
|
|
556
|
+
continue
|
|
557
|
+
new_rec = rec[:]
|
|
558
|
+
new_rec.seq = new_seq
|
|
559
|
+
trimmed_records.append(new_rec)
|
|
560
|
+
|
|
561
|
+
if trimmed_records :
|
|
562
|
+
with open(msa_in, "w") as msa_file :
|
|
563
|
+
for rec in trimmed_records:
|
|
564
|
+
msa_file.write(f">{rec.description}\n{rec.seq}\n")
|
|
565
|
+
|
|
566
|
+
#realign with mafft
|
|
567
|
+
cmd_mafft = f"mafft --quiet --anysymbol --thread 1 --parttree --retree 1 --maxiterate 0 {msa_in} > {aln_out}" #monothread
|
|
568
|
+
subprocess.run(cmd_mafft, shell=True, check=True)
|
|
569
|
+
|
|
570
|
+
#reformat a3m
|
|
571
|
+
cmd_reformat = f"reformat.pl fas a3m {aln_out} {msa_in}"
|
|
572
|
+
subprocess.run(cmd_reformat, shell=True, check=True)
|
|
573
|
+
cmd_rm_f = f"rm {aln_out}"
|
|
574
|
+
subprocess.run(cmd_rm_f, shell=True, check=True)
|
|
575
|
+
|
|
576
|
+
#delete \n
|
|
577
|
+
all_lines = ""
|
|
578
|
+
with open(msa_in, "r") as in_a3m:
|
|
579
|
+
seq, header = "", None
|
|
580
|
+
for line in in_a3m:
|
|
581
|
+
if line.startswith(">"):
|
|
582
|
+
if header:
|
|
583
|
+
all_lines += f"{header}\n{seq}\n"
|
|
584
|
+
header = line.strip()
|
|
585
|
+
seq = ""
|
|
586
|
+
else:
|
|
587
|
+
seq += line.strip()
|
|
588
|
+
if header:
|
|
589
|
+
all_lines += f"{header}\n{seq}\n"
|
|
590
|
+
with open(msa_in, "w") as out_a3m:
|
|
591
|
+
out_a3m.write(all_lines)
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def filter_signalP(file, Informations_dict, need_msa, need_pkl) :
|
|
596
|
+
"""
|
|
597
|
+
Filter proteins based on the presence of a signal peptide using SignalP results.
|
|
598
|
+
|
|
599
|
+
- Checks the first amino acid of each protein sequence to determine if a signal peptide is present (assumes sequences without SP start with 'M')
|
|
600
|
+
- Updates the result dictionary with "Signal_peptide" status
|
|
601
|
+
- Filters proteins from MSA and feature generation lists based on the user-specified SignalP
|
|
602
|
+
- Keeps all proteins if no filtering criterion is specified
|
|
603
|
+
|
|
604
|
+
Parameters :
|
|
605
|
+
----------
|
|
606
|
+
file : object of class File_proteins
|
|
607
|
+
Informations_dict : dict
|
|
608
|
+
need_msa : list
|
|
609
|
+
need_pkl : list
|
|
610
|
+
|
|
611
|
+
Returns :
|
|
612
|
+
----------
|
|
613
|
+
need_msa : list
|
|
614
|
+
need_pkl : list
|
|
615
|
+
"""
|
|
616
|
+
result_dict = file.get_result_dict()
|
|
617
|
+
possible_prey = file.get_possible_prey()
|
|
618
|
+
new_possible_prey = list()
|
|
619
|
+
SignalP = Informations_dict["Signal_peptide"]
|
|
620
|
+
for protein in possible_prey :
|
|
621
|
+
if SignalP == "None" :
|
|
622
|
+
new_possible_prey.append(protein)
|
|
623
|
+
elif SignalP == result_dict[protein]["Signal_peptide"] :
|
|
624
|
+
new_possible_prey.append(protein)
|
|
625
|
+
elif SignalP != result_dict[protein]["Signal_peptide"] :
|
|
626
|
+
result_dict[protein]["Reason_for_filtering"] = "Signal peptide : SignalP"
|
|
627
|
+
if protein in need_msa :
|
|
628
|
+
need_msa.remove(protein)
|
|
629
|
+
if protein in need_pkl :
|
|
630
|
+
need_pkl.remove(protein)
|
|
631
|
+
if SignalP != "None" :
|
|
632
|
+
logger.info("Protein preys remaining after SignalP filtering : " + str(len(new_possible_prey)))
|
|
633
|
+
file.set_possible_prey(new_possible_prey)
|
|
634
|
+
file.set_result_dict(result_dict)
|
|
635
|
+
return need_msa, need_pkl
|
|
636
|
+
|
|
637
|
+
def Make_all_MSA_coverage(file, Path_Pickle_Feature, baits, prey) :
|
|
638
|
+
"""
|
|
639
|
+
Generate MSA coverage plots for all proteins and create a shallow_MSA summary file.
|
|
640
|
+
|
|
641
|
+
- Computes sequence coverage and identity from the MSA
|
|
642
|
+
- Generates a coverage heatmap saved as PDF
|
|
643
|
+
- Determines if the MSA is "shallow" or missing (based on number of sequences)
|
|
644
|
+
- Updates the result dictionary and possible prey list accordingly
|
|
645
|
+
- Writes a summary of shallow MSAs to "log_file/shallow_MSA.txt"
|
|
646
|
+
|
|
647
|
+
Parameters :
|
|
648
|
+
----------
|
|
649
|
+
file : object of class File_proteins
|
|
650
|
+
Path_Pickle_Feature : string
|
|
651
|
+
baits : list
|
|
652
|
+
prey : list
|
|
653
|
+
"""
|
|
654
|
+
shallow_MSA = str()
|
|
655
|
+
result_dict = file.get_result_dict()
|
|
656
|
+
all_prot = prey
|
|
657
|
+
if baits != [''] :
|
|
658
|
+
for bait in baits :
|
|
659
|
+
all_prot.append(bait)
|
|
660
|
+
new_possible_prey = list()
|
|
661
|
+
for prot in all_prot :
|
|
662
|
+
if Path(f'{Path_Pickle_Feature}/{prot}_coverage.pdf').exists() == False :
|
|
663
|
+
#Make coverage plot
|
|
664
|
+
pre_feature_dict = pickle.load(open(f"{Path_Pickle_Feature}/{prot}.pkl","rb"))
|
|
665
|
+
feature_dict = pre_feature_dict.feature_dict
|
|
666
|
+
msa = feature_dict['msa']
|
|
667
|
+
seqid = (np.array(msa[0] == msa).mean(-1))
|
|
668
|
+
seqid_sort = seqid.argsort()
|
|
669
|
+
non_gaps = (msa != 21).astype(float)
|
|
670
|
+
non_gaps[non_gaps == 0] = np.nan
|
|
671
|
+
final = non_gaps[seqid_sort] * seqid[seqid_sort, None]
|
|
672
|
+
plt.figure(figsize=(14, 4), dpi=100)
|
|
673
|
+
plt.subplot(1, 2, 1)
|
|
674
|
+
plt.title(f"Sequence coverage ({prot})")
|
|
675
|
+
plt.imshow(final, interpolation="nearest", aspect="auto", cmap="rainbow_r", vmin=0, vmax=1, origin="lower")
|
|
676
|
+
plt.plot((msa != 21).sum(0), color="black")
|
|
677
|
+
plt.xlim(-0.5, msa.shape[1] - 0.5)
|
|
678
|
+
plt.ylim(-0.5, msa.shape[0] - 0.5)
|
|
679
|
+
plt.colorbar(label="Sequence identity to query", )
|
|
680
|
+
plt.xlabel("Positions")
|
|
681
|
+
plt.ylabel("Sequences")
|
|
682
|
+
plt.savefig(f"{Path_Pickle_Feature}/{prot+('_' if prot else '')}coverage.pdf")
|
|
683
|
+
plt.close()
|
|
684
|
+
#Evaluate MSA depth
|
|
685
|
+
with open(f'{Path_Pickle_Feature}/{prot}.a3m') as msa_f :
|
|
686
|
+
line_msa = sum(1 for _ in msa_f)
|
|
687
|
+
real_nb_msa = line_msa/2
|
|
688
|
+
if real_nb_msa <= 100 and prot not in baits :
|
|
689
|
+
if real_nb_msa < 2 :
|
|
690
|
+
shallow_MSA += prot + " : " + str(int(real_nb_msa)) + " sequences\n"
|
|
691
|
+
result_dict[prot]["shallow_MSA"] = "No MSA"
|
|
692
|
+
result_dict[prot]["Reason_for_filtering"] = "No MSA"
|
|
693
|
+
else :
|
|
694
|
+
shallow_MSA += prot + " : " + str(int(real_nb_msa)) + " sequences\n"
|
|
695
|
+
result_dict[prot]["shallow_MSA"] = "Shallow MSA"
|
|
696
|
+
new_possible_prey.append(prot)
|
|
697
|
+
if real_nb_msa > 100 and prot not in baits :
|
|
698
|
+
new_possible_prey.append(prot)
|
|
699
|
+
if real_nb_msa <= 100 and prot in baits :
|
|
700
|
+
logger.warning(f"This bait : {prot} have a shallow MSA with {int(real_nb_msa)} sequences. Predicted interactions involving this protein should be interpreted with caution.")
|
|
701
|
+
with open("log_file/shallow_MSA.txt", "w") as MSA_file :
|
|
702
|
+
MSA_file.write(shallow_MSA)
|
|
703
|
+
file.set_possible_prey(new_possible_prey)
|
|
704
|
+
file.set_result_dict(result_dict)
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
|
|
708
|
+
def filter_lenght(file, Informations_dict, need_msa, need_pkl, need_DeepLoc) :
|
|
709
|
+
"""
|
|
710
|
+
Filter proteins based on their sequence length and update the list of possible preys.
|
|
711
|
+
|
|
712
|
+
- Uses the full sequence (including signal peptide) to evaluate protein length
|
|
713
|
+
- Filters proteins that are either too short or too long according to user-defined thresholds
|
|
714
|
+
- Updates the result dictionary with a reason for filtering
|
|
715
|
+
- Removes filtered proteins from MSA, pickle, and DeepLoc processing lists
|
|
716
|
+
- Updates the File_proteins object with the new prey list
|
|
717
|
+
|
|
718
|
+
Parameters :
|
|
719
|
+
----------
|
|
720
|
+
file : object of class File_proteins
|
|
721
|
+
Informations_dict : dict
|
|
722
|
+
need_msa : list
|
|
723
|
+
need_pkl : list
|
|
724
|
+
need_DeepLoc : list
|
|
725
|
+
|
|
726
|
+
Returns :
|
|
727
|
+
----------
|
|
728
|
+
need_msa : list
|
|
729
|
+
need_pkl : list
|
|
730
|
+
need_DeepLoc : list
|
|
731
|
+
"""
|
|
732
|
+
result_dict = file.get_result_dict()
|
|
733
|
+
sequence_dict = file.get_proteins_sequence_SP() #use sequence with SP for lenght filtering
|
|
734
|
+
possible_prey = file.get_possible_prey()
|
|
735
|
+
if Informations_dict["Max_protein_lenght"] == "" : #not set by default, depend of GPU memory
|
|
736
|
+
max_lenght = 100000
|
|
737
|
+
else :
|
|
738
|
+
max_lenght = int(Informations_dict["Max_protein_lenght"])
|
|
739
|
+
min_lenght = int(Informations_dict["Min_protein_lenght"]) #set by default to 20
|
|
740
|
+
new_possible_prey = list()
|
|
741
|
+
for protein in possible_prey :
|
|
742
|
+
if len(sequence_dict[protein]) < max_lenght and len(sequence_dict[protein]) > min_lenght :
|
|
743
|
+
new_possible_prey.append(protein)
|
|
744
|
+
else :
|
|
745
|
+
result_dict[protein]["Reason_for_filtering"] = "Lenght filtering"
|
|
746
|
+
if protein in need_msa :
|
|
747
|
+
need_msa.remove(protein)
|
|
748
|
+
if protein in need_pkl :
|
|
749
|
+
need_pkl.remove(protein)
|
|
750
|
+
if protein in need_DeepLoc :
|
|
751
|
+
need_DeepLoc.remove(protein)
|
|
752
|
+
logger.info("Protein preys remaining after lenght filtering : " + str(len(new_possible_prey)))
|
|
753
|
+
file.set_possible_prey(new_possible_prey)
|
|
754
|
+
file.set_result_dict(result_dict)
|
|
755
|
+
return(need_msa, need_pkl, need_DeepLoc)
|
|
756
|
+
|
|
757
|
+
|
|
758
|
+
def filter_deeploc(file, Informations_dict, need_msa, need_pkl) :
|
|
759
|
+
"""
|
|
760
|
+
Filter proteins based on predicted cellular localization and update the prey list as well as the MSA and pickle processing lists.
|
|
761
|
+
|
|
762
|
+
- Compares each protein's predicted DeepLoc localization(s) with the user-specified allowed localizations
|
|
763
|
+
- Keeps proteins matching at least one allowed localization
|
|
764
|
+
- Marks proteins that do not match as filtered in the result dictionary
|
|
765
|
+
- Updates the need_msa and need_pkl lists to include only proteins that pass the filter or are baits
|
|
766
|
+
|
|
767
|
+
Parameters :
|
|
768
|
+
----------
|
|
769
|
+
file : object of class File_proteins
|
|
770
|
+
Informations_dict : dict
|
|
771
|
+
need_msa : list
|
|
772
|
+
need_pkl : list
|
|
773
|
+
|
|
774
|
+
Returns :
|
|
775
|
+
----------
|
|
776
|
+
new_need_msa : list
|
|
777
|
+
new_need_pkl : list
|
|
778
|
+
"""
|
|
779
|
+
localisation = Informations_dict["DeepLoc"].split(",")
|
|
780
|
+
localisation = [loc.strip() for loc in localisation]
|
|
781
|
+
deeploc = file.get_deeploc()
|
|
782
|
+
possible_baits = Informations_dict["Interact_with"]
|
|
783
|
+
possible_prey = file.get_possible_prey()
|
|
784
|
+
result_dict = file.get_result_dict()
|
|
785
|
+
new_possible_prey = list()
|
|
786
|
+
new_need_msa = list()
|
|
787
|
+
new_need_pkl = list()
|
|
788
|
+
for protein in possible_prey :
|
|
789
|
+
for loc in deeploc[protein].split("|") :
|
|
790
|
+
if loc in localisation :
|
|
791
|
+
new_possible_prey.append(protein)
|
|
792
|
+
break #stop the loop if one localisation is correct
|
|
793
|
+
if protein not in new_possible_prey :
|
|
794
|
+
result_dict[protein]["Reason_for_filtering"] = "Cellular localisation : DeepLoc"
|
|
795
|
+
for prot_msa in need_msa :
|
|
796
|
+
if prot_msa in new_possible_prey or prot_msa in possible_baits :
|
|
797
|
+
new_need_msa.append(prot_msa)
|
|
798
|
+
for prot_pkl in need_pkl :
|
|
799
|
+
if prot_pkl in new_possible_prey or prot_pkl in possible_baits :
|
|
800
|
+
new_need_pkl.append(prot_pkl)
|
|
801
|
+
logger.info("Protein prey remaining after DeepLoc filtering : " + str(len(new_possible_prey)))
|
|
802
|
+
file.set_possible_prey(new_possible_prey)
|
|
803
|
+
file.set_result_dict(result_dict)
|
|
804
|
+
return(new_need_msa, new_need_pkl)
|
|
805
|
+
|
|
806
|
+
|
|
807
|
+
def Generate_scripts(file, Informations_dict, Interaction_file, bait) :
|
|
808
|
+
"""
|
|
809
|
+
Prepare the list of interaction jobs, taking into account protein lengths and GPU VRAM constraints to avoid out-of-memory (OOM) errors.
|
|
810
|
+
|
|
811
|
+
- Estimates the maximum number of amino acids allowed based on available GPU VRAM
|
|
812
|
+
- Builds interaction jobs (PPI or homo-oligomeric) only if they fit in memory
|
|
813
|
+
- Skips interactions already computed
|
|
814
|
+
- Records interactions that are too large for the GPU in a log file
|
|
815
|
+
|
|
816
|
+
Parameters :
|
|
817
|
+
----------
|
|
818
|
+
file : object of class File_proteins
|
|
819
|
+
Informations_dict : dict
|
|
820
|
+
Interaction_file : str
|
|
821
|
+
bait : str
|
|
822
|
+
|
|
823
|
+
Returns :
|
|
824
|
+
----------
|
|
825
|
+
job_with_vram_length : list of tuple
|
|
826
|
+
"""
|
|
827
|
+
job_with_vram_length = [] # [(job_str, int_length)]
|
|
828
|
+
OOM_int = ""
|
|
829
|
+
result_dict = file.get_result_dict()
|
|
830
|
+
AF_version = Informations_dict["AlphaFold"]
|
|
831
|
+
possible_prey = file.get_possible_prey()
|
|
832
|
+
lenght_prot = file.get_lenght_prot()
|
|
833
|
+
regions = Informations_dict["Regions"]
|
|
834
|
+
|
|
835
|
+
# Estimate max amino acids based on GPU VRAM (choose first GPU as reference)
|
|
836
|
+
pynvml.nvmlInit()
|
|
837
|
+
handle = pynvml.nvmlDeviceGetHandleByIndex(0)
|
|
838
|
+
mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
|
|
839
|
+
vram = (mem_info.total / 1024**2) * 0.001 # GiB
|
|
840
|
+
max_aa = int((0.0000627 + math.sqrt(0.0000627**2 - 4*0.00000332*(3.8 - vram))) / (2*0.00000332))
|
|
841
|
+
pynvml.nvmlShutdown()
|
|
842
|
+
|
|
843
|
+
if Interaction_file == "PPI_int" or Interaction_file == "Compounds" :
|
|
844
|
+
# Determine bait length and region
|
|
845
|
+
complexe = "," in bait
|
|
846
|
+
if complexe :
|
|
847
|
+
save_multimer = bait
|
|
848
|
+
bait_file = save_multimer.replace(",", "_and_")
|
|
849
|
+
bait_for_job = save_multimer.replace(",", ";")
|
|
850
|
+
lenght = sum(lenght_prot[prot] for prot in save_multimer.split(","))
|
|
851
|
+
for prot in save_multimer.split(",") :
|
|
852
|
+
if regions[prot] != "0-0":
|
|
853
|
+
start, end = int(regions[prot].split("-")[0]), int(regions[prot].split("-")[1])
|
|
854
|
+
bait_file = bait_file.replace(prot, f"{prot}_{start}-{end}")
|
|
855
|
+
bait_for_job = bait_for_job.replace(prot, f"{prot},{start}-{end}")
|
|
856
|
+
else :
|
|
857
|
+
lenght = lenght_prot[bait]
|
|
858
|
+
if regions[bait] != "0-0" :
|
|
859
|
+
start, end = int(regions[bait].split("-")[0]), int(regions[bait].split("-")[1])
|
|
860
|
+
bait_file = f"{bait}_{start}-{end}"
|
|
861
|
+
bait_for_job = f"{bait},{start}-{end}"
|
|
862
|
+
else :
|
|
863
|
+
bait_file = bait
|
|
864
|
+
bait_for_job = bait
|
|
865
|
+
|
|
866
|
+
# Build job list
|
|
867
|
+
if Interaction_file == "PPI_int" :
|
|
868
|
+
copy_possible_prey = copy.deepcopy(possible_prey) # To avoid modifying the list while iterating
|
|
869
|
+
for prey in copy_possible_prey :
|
|
870
|
+
int_lenght = lenght + lenght_prot[prey]
|
|
871
|
+
|
|
872
|
+
# Check if model already exists
|
|
873
|
+
if AF_version == "3" :
|
|
874
|
+
path1 = glob.glob(f"./result_PPI_int/{bait_file}_and_{prey}/*_model.cif")
|
|
875
|
+
path2 = glob.glob(f"./result_PPI_int/{prey}_and_{bait_file}/*_model.cif")
|
|
876
|
+
else : # AF_version == "2"
|
|
877
|
+
path1 = glob.glob(f"./result_PPI_int/{bait_file}_and_{prey}/ranked_0.pdb")
|
|
878
|
+
path2 = glob.glob(f"./result_PPI_int/{prey}_and_{bait_file}/ranked_0.pdb")
|
|
879
|
+
|
|
880
|
+
if len(path1) == 0 and len(path2) == 0 :
|
|
881
|
+
if int_lenght <= max_aa :
|
|
882
|
+
job_str = f"{bait_for_job};{prey}\n"
|
|
883
|
+
vram_lenght = 3.8 + (-0.0000627) * int_lenght + 0.00000332 * int_lenght**2
|
|
884
|
+
job_with_vram_length.append((job_str, vram_lenght))
|
|
885
|
+
else :
|
|
886
|
+
OOM_int += f"{bait_for_job};{prey}\n"
|
|
887
|
+
result_dict[prey][f"Reason_for_filtering"] = "Interaction too large for your GPU, possible prey"
|
|
888
|
+
possible_prey.remove(prey)
|
|
889
|
+
if Interaction_file == "Compounds" :
|
|
890
|
+
Compounds = file.get_compounds()
|
|
891
|
+
vram_lenght = 1.9 + (-0.0000627) * lenght + 0.00000332 * lenght**2
|
|
892
|
+
for compound in Compounds.keys() :
|
|
893
|
+
job_str = f"{bait_for_job};{compound}\n"
|
|
894
|
+
path = glob.glob(f"./result_Compounds/{bait_file}_and_{compound}/*_model.cif")
|
|
895
|
+
if len(path) == 0 :
|
|
896
|
+
if os.path.isdir(f"./result_Compounds/{bait_file}_and_{compound}") : #if dir exist and model not exist, rm dir
|
|
897
|
+
os.system(f"rm -r ./result_Compounds/{bait_file}_and_{compound}")
|
|
898
|
+
job_with_vram_length.append((f"{job_str}", vram_lenght)) #consider no vram for compound, only for protein bait
|
|
899
|
+
|
|
900
|
+
if Interaction_file == "homo_int" :
|
|
901
|
+
nbr_oligo = Informations_dict.get("Homo-oligomer", 2)
|
|
902
|
+
for prey in possible_prey :
|
|
903
|
+
int_lenght = lenght_prot[prey] * int(nbr_oligo)
|
|
904
|
+
if AF_version == "3" :
|
|
905
|
+
path = glob.glob(f"./result_homo_int/{prey}_homo_{nbr_oligo}er/*_model.cif")
|
|
906
|
+
else :
|
|
907
|
+
path = glob.glob(f"./result_homo_int/{prey}_homo_{nbr_oligo}er/ranked_0.pdb")
|
|
908
|
+
if len(path) == 0 :
|
|
909
|
+
if int_lenght <= max_aa :
|
|
910
|
+
if AF_version == "3" :
|
|
911
|
+
job_str = f"{prey}_af3_input.json:{nbr_oligo}\n"
|
|
912
|
+
if AF_version == "2" :
|
|
913
|
+
job_str = f"{prey}:{nbr_oligo}\n"
|
|
914
|
+
vram_lenght = 3.8 + (-0.0000627) * int_lenght + 0.00000332 * int_lenght**2
|
|
915
|
+
job_with_vram_length.append((job_str, vram_lenght))
|
|
916
|
+
else :
|
|
917
|
+
OOM_int += f"{prey}:{nbr_oligo}\n"
|
|
918
|
+
result_dict[prey]["Reason_for_filtering"] = "Homo-oligomer too large for your GPU"
|
|
919
|
+
possible_prey.remove(prey)
|
|
920
|
+
|
|
921
|
+
|
|
922
|
+
# Classify job_list in function of int lenght
|
|
923
|
+
job_with_vram_length.sort(key=lambda x: x[1], reverse=True)
|
|
924
|
+
|
|
925
|
+
|
|
926
|
+
# Save OOM interactions
|
|
927
|
+
with open("log_file/OOM_interactions.txt", "w") as OOM_file :
|
|
928
|
+
OOM_file.write(OOM_int)
|
|
929
|
+
|
|
930
|
+
file.set_possible_prey(possible_prey)
|
|
931
|
+
file.set_result_dict(result_dict)
|
|
932
|
+
|
|
933
|
+
return job_with_vram_length
|
|
934
|
+
|
|
935
|
+
|
|
936
|
+
def Generate_first_batch(job_with_vram_length, GPU, multi_job_per_gpu) :
|
|
937
|
+
if not job_with_vram_length :
|
|
938
|
+
return
|
|
939
|
+
|
|
940
|
+
pynvml.nvmlInit()
|
|
941
|
+
handle = pynvml.nvmlDeviceGetHandleByIndex(int(GPU[0]))
|
|
942
|
+
mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
|
|
943
|
+
max_vram = mem_info.total / 1024**3 #Gb
|
|
944
|
+
pynvml.nvmlShutdown()
|
|
945
|
+
|
|
946
|
+
gpu_vram_used = {gpu: 0.0 for gpu in GPU}
|
|
947
|
+
gpu_jobs = []
|
|
948
|
+
|
|
949
|
+
multi_job_per_gpu = False if multi_job_per_gpu == "False" else True
|
|
950
|
+
|
|
951
|
+
pending = list(job_with_vram_length)
|
|
952
|
+
|
|
953
|
+
for interaction, job_vram in pending :
|
|
954
|
+
|
|
955
|
+
sorted_gpus = sorted(GPU, key=lambda g: gpu_vram_used[g])
|
|
956
|
+
|
|
957
|
+
launched = False
|
|
958
|
+
|
|
959
|
+
for gpu_id in sorted_gpus :
|
|
960
|
+
|
|
961
|
+
free = max_vram - gpu_vram_used[gpu_id]
|
|
962
|
+
|
|
963
|
+
if job_vram <= free and (multi_job_per_gpu or len(gpu_jobs[gpu_id]) == 0) :
|
|
964
|
+
|
|
965
|
+
gpu_jobs.append(interaction.split(";")[-1].strip("\n"))
|
|
966
|
+
gpu_vram_used[gpu_id] += job_vram
|
|
967
|
+
|
|
968
|
+
launched = True
|
|
969
|
+
break
|
|
970
|
+
|
|
971
|
+
if not launched :
|
|
972
|
+
continue
|
|
973
|
+
|
|
974
|
+
return gpu_jobs, gpu_vram_used
|
|
975
|
+
|
|
976
|
+
def prioritize_by_vram_fit(jobs, GPU, max_per_batch=3):
|
|
977
|
+
|
|
978
|
+
import pynvml
|
|
979
|
+
|
|
980
|
+
pynvml.nvmlInit()
|
|
981
|
+
handle = pynvml.nvmlDeviceGetHandleByIndex(int(GPU[0]))
|
|
982
|
+
max_vram = pynvml.nvmlDeviceGetMemoryInfo(handle).total / 1024**3
|
|
983
|
+
pynvml.nvmlShutdown()
|
|
984
|
+
|
|
985
|
+
jobs = sorted(jobs, key=lambda x: x[1], reverse=True)
|
|
986
|
+
|
|
987
|
+
remaining = jobs.copy()
|
|
988
|
+
all_batches = []
|
|
989
|
+
|
|
990
|
+
while remaining:
|
|
991
|
+
|
|
992
|
+
batch = []
|
|
993
|
+
used_vram = 0.0
|
|
994
|
+
|
|
995
|
+
# on tente plusieurs passes pour remplir correctement
|
|
996
|
+
changed = True
|
|
997
|
+
|
|
998
|
+
while changed and len(batch) < max_per_batch:
|
|
999
|
+
changed = False
|
|
1000
|
+
|
|
1001
|
+
for i in range(len(remaining)):
|
|
1002
|
+
|
|
1003
|
+
job, vram = remaining[i]
|
|
1004
|
+
|
|
1005
|
+
if len(batch) < max_per_batch and used_vram + vram <= max_vram:
|
|
1006
|
+
batch.append(job.split(";")[-1].strip("\n"))
|
|
1007
|
+
used_vram += vram
|
|
1008
|
+
remaining.pop(i)
|
|
1009
|
+
changed = True
|
|
1010
|
+
break
|
|
1011
|
+
|
|
1012
|
+
all_batches.append(batch)
|
|
1013
|
+
|
|
1014
|
+
return all_batches
|
|
1015
|
+
|
|
1016
|
+
def Generate_3D_model(HInt_object, CPU, multi_scoring, Informations_dict, interaction_type, job_with_vram_length, GPU, multi_job_per_gpu, seq_bait={}, compounds_dict={}) :
|
|
1017
|
+
"""
|
|
1018
|
+
Generate 3D models using multiple GPUs and multiprocessing.
|
|
1019
|
+
|
|
1020
|
+
Parameters :
|
|
1021
|
+
----------
|
|
1022
|
+
Informations_dict : dict
|
|
1023
|
+
interaction_files : str
|
|
1024
|
+
job_with_vram_length : list of tuple
|
|
1025
|
+
GPU : list
|
|
1026
|
+
multi_job_per_gpu : str
|
|
1027
|
+
seq_bait : dict
|
|
1028
|
+
compounds_dict : dict
|
|
1029
|
+
"""
|
|
1030
|
+
if not job_with_vram_length :
|
|
1031
|
+
return
|
|
1032
|
+
|
|
1033
|
+
pynvml.nvmlInit()
|
|
1034
|
+
handle = pynvml.nvmlDeviceGetHandleByIndex(int(GPU[0]))
|
|
1035
|
+
mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
|
|
1036
|
+
max_vram = mem_info.total / 1024**3 #Gb
|
|
1037
|
+
pynvml.nvmlShutdown()
|
|
1038
|
+
|
|
1039
|
+
stop_flag = multiprocessing.Event()
|
|
1040
|
+
monitor = multiprocessing.Process(
|
|
1041
|
+
target=monitor_vram,
|
|
1042
|
+
args=(GPU, stop_flag)
|
|
1043
|
+
)
|
|
1044
|
+
monitor.start()
|
|
1045
|
+
|
|
1046
|
+
with open("log_file/All_PPI_jobs.txt", "w") as f : #write complete script for informations
|
|
1047
|
+
for job, _ in job_with_vram_length :
|
|
1048
|
+
f.write(job)
|
|
1049
|
+
manager(jobs_pending=list(job_with_vram_length),
|
|
1050
|
+
HInt_object=HInt_object,
|
|
1051
|
+
CPU=CPU,
|
|
1052
|
+
multi_scoring=multi_scoring,
|
|
1053
|
+
Informations_dict=Informations_dict,
|
|
1054
|
+
GPU=GPU,
|
|
1055
|
+
max_vram=max_vram,
|
|
1056
|
+
interaction_type=interaction_type,
|
|
1057
|
+
multi_job_per_gpu=multi_job_per_gpu,
|
|
1058
|
+
seq_bait=seq_bait,
|
|
1059
|
+
compounds_dict=compounds_dict
|
|
1060
|
+
)
|
|
1061
|
+
|
|
1062
|
+
stop_flag.set()
|
|
1063
|
+
monitor.join()
|
|
1064
|
+
|
|
1065
|
+
def manager(jobs_pending, HInt_object, CPU, multi_scoring, Informations_dict, GPU, max_vram, interaction_type, multi_job_per_gpu, seq_bait, compounds_dict) :
|
|
1066
|
+
"""
|
|
1067
|
+
Manage repartition of jobs on GPUs, monitor VRAM usage, and handle job completion.
|
|
1068
|
+
|
|
1069
|
+
Parameters :
|
|
1070
|
+
----------
|
|
1071
|
+
jobs_pending : list
|
|
1072
|
+
GPU : list
|
|
1073
|
+
max_vram : float
|
|
1074
|
+
Path_AlphaFold_Data : str
|
|
1075
|
+
Path_Pickle_Feature : str
|
|
1076
|
+
interaction_type : str
|
|
1077
|
+
AF_version : str
|
|
1078
|
+
multi_job_per_gpu : str
|
|
1079
|
+
seq_bait : dict
|
|
1080
|
+
Baits : list
|
|
1081
|
+
compounds_dict : dict
|
|
1082
|
+
"""
|
|
1083
|
+
AF_version = Informations_dict["AlphaFold"]
|
|
1084
|
+
Path_AlphaFold_Data = Informations_dict["Path_AlphaFold_Data"]
|
|
1085
|
+
Path_Pickle_Feature = Informations_dict["Path_Pickle_Feature"]
|
|
1086
|
+
Baits = Informations_dict["Interact_with"]
|
|
1087
|
+
compound = None
|
|
1088
|
+
manager_proc = multiprocessing.Manager()
|
|
1089
|
+
result_queue = manager_proc.Queue()
|
|
1090
|
+
multi_job_per_gpu = False if multi_job_per_gpu == "False" else True
|
|
1091
|
+
gpu_vram_used = {gpu: 0.0 for gpu in GPU}
|
|
1092
|
+
jobs_running = {gpu: [] for gpu in GPU} # {gpu_id: {job_str: process}}
|
|
1093
|
+
|
|
1094
|
+
while jobs_pending or any(jobs_running.values()):
|
|
1095
|
+
|
|
1096
|
+
while not result_queue.empty() :
|
|
1097
|
+
status, job, gpu_id, vram = result_queue.get()[:4]
|
|
1098
|
+
gpu_vram_used[gpu_id] -= vram
|
|
1099
|
+
for i, (j, p) in enumerate(jobs_running[gpu_id]):
|
|
1100
|
+
if j == job:
|
|
1101
|
+
p.join()
|
|
1102
|
+
jobs_running[gpu_id].pop(i)
|
|
1103
|
+
Score_interaction(HInt_object, Informations_dict, CPU, interaction_type, job, multi_scoring, "")
|
|
1104
|
+
HInt_object.Make_save_dict()
|
|
1105
|
+
break
|
|
1106
|
+
|
|
1107
|
+
launched = False
|
|
1108
|
+
|
|
1109
|
+
for interaction, job_vram in list(jobs_pending) :
|
|
1110
|
+
sorted_gpus = sorted(GPU, key=lambda g: gpu_vram_used[g]) #make PPI on GPU with minimum vram use
|
|
1111
|
+
if interaction_type == "Compounds" :
|
|
1112
|
+
compound = compounds_dict[interaction.strip("\n").split(";")[-1]] #to just select one compound for each interaction
|
|
1113
|
+
for gpu_id in sorted_gpus :
|
|
1114
|
+
|
|
1115
|
+
free = max_vram - gpu_vram_used[gpu_id]
|
|
1116
|
+
if job_vram <= free and (multi_job_per_gpu or len(jobs_running[gpu_id]) == 0) :
|
|
1117
|
+
p = multiprocessing.Process(target=gpu_job_runner,
|
|
1118
|
+
args=(gpu_id,
|
|
1119
|
+
interaction,
|
|
1120
|
+
job_vram,
|
|
1121
|
+
result_queue,
|
|
1122
|
+
Path_AlphaFold_Data,
|
|
1123
|
+
Path_Pickle_Feature,
|
|
1124
|
+
interaction_type,
|
|
1125
|
+
AF_version,
|
|
1126
|
+
seq_bait,
|
|
1127
|
+
Baits,
|
|
1128
|
+
compound
|
|
1129
|
+
),
|
|
1130
|
+
)
|
|
1131
|
+
|
|
1132
|
+
p.start()
|
|
1133
|
+
|
|
1134
|
+
gpu_vram_used[gpu_id] += job_vram
|
|
1135
|
+
jobs_running[gpu_id].append((interaction, p))
|
|
1136
|
+
jobs_pending.remove((interaction, job_vram))
|
|
1137
|
+
launched = True
|
|
1138
|
+
break
|
|
1139
|
+
|
|
1140
|
+
if launched :
|
|
1141
|
+
break
|
|
1142
|
+
if not launched :
|
|
1143
|
+
time.sleep(1)
|
|
1144
|
+
|
|
1145
|
+
|
|
1146
|
+
def gpu_job_runner(gpu_id, interaction_file, vram, result_queue, Path_AlphaFold_Data, Path_Pickle_Feature, interaction_type, AF_version, seq_bait, Baits, compound) :
|
|
1147
|
+
"""
|
|
1148
|
+
Run a single AlphaFold job on a specified GPU, monitor its completion, and report results.
|
|
1149
|
+
|
|
1150
|
+
Parameters :
|
|
1151
|
+
----------
|
|
1152
|
+
gpu_id : int
|
|
1153
|
+
interaction_file : str
|
|
1154
|
+
vram : float
|
|
1155
|
+
result_queue : multiprocessing.Queue
|
|
1156
|
+
Path_AlphaFold_Data : str
|
|
1157
|
+
Path_Pickle_Feature : str
|
|
1158
|
+
interaction_type : str
|
|
1159
|
+
AF_version : str
|
|
1160
|
+
seq_bait : dict
|
|
1161
|
+
Baits : list
|
|
1162
|
+
compound : str
|
|
1163
|
+
"""
|
|
1164
|
+
env = os.environ.copy()
|
|
1165
|
+
env['CUDA_VISIBLE_DEVICES'] = str(gpu_id)
|
|
1166
|
+
env['XLA_PYTHON_CLIENT_PREALLOCATE'] = 'false'
|
|
1167
|
+
env['TF_FORCE_UNIFIED_MEMORY'] = 'true'
|
|
1168
|
+
env['XLA_FLAGS'] = '--xla_gpu_enable_triton_gemm=false'
|
|
1169
|
+
|
|
1170
|
+
if interaction_type == "PPI_int" or interaction_type == "Compounds" :
|
|
1171
|
+
prot_int = interaction_file.strip("\n").replace(";", "_and_")
|
|
1172
|
+
if interaction_type == "homo_int" :
|
|
1173
|
+
prot_int = interaction_file.strip("\n").replace(":", "_homo_")
|
|
1174
|
+
prot_int += "er"
|
|
1175
|
+
prot_int = prot_int.replace(",","_")
|
|
1176
|
+
|
|
1177
|
+
file_name = f"{interaction_type}_GPU_{prot_int}.txt"
|
|
1178
|
+
|
|
1179
|
+
if interaction_type == "Compounds" : #Write json file to use AF3 native
|
|
1180
|
+
random_seed = random.randrange(2**32 - 1)
|
|
1181
|
+
sequences = []
|
|
1182
|
+
ligands = []
|
|
1183
|
+
i = -1
|
|
1184
|
+
for obj in interaction_file.strip("\n").split(";") :
|
|
1185
|
+
i += 1
|
|
1186
|
+
if "," in obj :
|
|
1187
|
+
obj = obj.split(",")[0]
|
|
1188
|
+
if obj in Baits : #it's protein
|
|
1189
|
+
with open(f"{Path_Pickle_Feature}/{obj}.a3m", "r") as a3m_f :
|
|
1190
|
+
all_lines = a3m_f.read()
|
|
1191
|
+
sequences.append({
|
|
1192
|
+
"protein": {
|
|
1193
|
+
"id": string.ascii_uppercase[i],
|
|
1194
|
+
"sequence": seq_bait[obj],
|
|
1195
|
+
"unpairedMsa": all_lines,
|
|
1196
|
+
"pairedMsa": all_lines,
|
|
1197
|
+
"templates": []
|
|
1198
|
+
}
|
|
1199
|
+
})
|
|
1200
|
+
|
|
1201
|
+
else : # it's LIGAND (SMILES)
|
|
1202
|
+
ligands.append({
|
|
1203
|
+
"ligand": {
|
|
1204
|
+
"id": string.ascii_uppercase[i],
|
|
1205
|
+
"smiles": compound
|
|
1206
|
+
}
|
|
1207
|
+
})
|
|
1208
|
+
af3_input = {
|
|
1209
|
+
"name": prot_int,
|
|
1210
|
+
"dialect": "alphafold3",
|
|
1211
|
+
"version": 3,
|
|
1212
|
+
"modelSeeds": [random_seed],
|
|
1213
|
+
"sequences": sequences + ligands
|
|
1214
|
+
}
|
|
1215
|
+
file_name = file_name.replace(".txt", ".json")
|
|
1216
|
+
with open(f"log_file/{file_name}", "w") as f:
|
|
1217
|
+
json.dump(af3_input, f, indent=2)
|
|
1218
|
+
try :
|
|
1219
|
+
if interaction_type != "Compounds" :
|
|
1220
|
+
with open(f"log_file/{file_name}", "w") as f :
|
|
1221
|
+
f.write(interaction_file)
|
|
1222
|
+
|
|
1223
|
+
if AF_version == "2" :
|
|
1224
|
+
cmd = (
|
|
1225
|
+
"run_multimer_jobs.py --mode=custom "
|
|
1226
|
+
"--num_cycle=3 "
|
|
1227
|
+
"--num_predictions_per_model=1 "
|
|
1228
|
+
"--compress_result_pickles=True "
|
|
1229
|
+
f"--output_path=./result_{interaction_type} "
|
|
1230
|
+
f"--data_dir={Path_AlphaFold_Data} "
|
|
1231
|
+
f"--protein_lists=log_file/{file_name} "
|
|
1232
|
+
f"--monomer_objects_dir={Path_Pickle_Feature} "
|
|
1233
|
+
"--remove_keys_from_pickles=False"
|
|
1234
|
+
)
|
|
1235
|
+
if AF_version == "3" and interaction_type != "Compounds" :
|
|
1236
|
+
cmd = (
|
|
1237
|
+
"run_multimer_jobs.py --mode=custom "
|
|
1238
|
+
f"--output_path=./result_{interaction_type} "
|
|
1239
|
+
f"--data_dir={Path_AlphaFold_Data} "
|
|
1240
|
+
f"--protein_lists=log_file/{file_name} "
|
|
1241
|
+
f"--monomer_objects_dir={Path_Pickle_Feature} "
|
|
1242
|
+
"--fold_backend=alphafold3 "
|
|
1243
|
+
"--use_ap_style=True "
|
|
1244
|
+
)
|
|
1245
|
+
|
|
1246
|
+
if AF_version == "3" and interaction_type == "Compounds" :
|
|
1247
|
+
cmd = (f"python ./alphafold3/run_alphafold.py --output_dir=./result_{interaction_type} --db_dir={Path_AlphaFold_Data} --model_dir={Path_AlphaFold_Data} --json_path=log_file/{file_name}")
|
|
1248
|
+
logger.info(f"[GPU {gpu_id}] Starting {interaction_file}")
|
|
1249
|
+
proc = subprocess.Popen(
|
|
1250
|
+
cmd,
|
|
1251
|
+
shell=True,
|
|
1252
|
+
env=env,
|
|
1253
|
+
preexec_fn=os.setsid
|
|
1254
|
+
)
|
|
1255
|
+
|
|
1256
|
+
while True :
|
|
1257
|
+
|
|
1258
|
+
ret = proc.poll()
|
|
1259
|
+
if ret is not None:
|
|
1260
|
+
if ret != 0 :
|
|
1261
|
+
raise RuntimeError(f"Crash ({ret})")
|
|
1262
|
+
break
|
|
1263
|
+
|
|
1264
|
+
time.sleep(0.5)
|
|
1265
|
+
|
|
1266
|
+
result_queue.put(("done", interaction_file, gpu_id, vram))
|
|
1267
|
+
os.system(f"rm log_file/{file_name}")
|
|
1268
|
+
logger.info(f"[GPU {gpu_id}] Finished {interaction_file}")
|
|
1269
|
+
|
|
1270
|
+
except KeyboardInterrupt :
|
|
1271
|
+
kill_hint_processes(proc)
|
|
1272
|
+
|
|
1273
|
+
|
|
1274
|
+
|
|
1275
|
+
def monitor_vram(GPU, stop_flag) :
|
|
1276
|
+
"""
|
|
1277
|
+
Monitor VRAM usage for given GPUs every `interval` seconds.
|
|
1278
|
+
"""
|
|
1279
|
+
|
|
1280
|
+
pynvml.nvmlInit()
|
|
1281
|
+
handles = {int(gpu): pynvml.nvmlDeviceGetHandleByIndex(int(gpu)) for gpu in GPU}
|
|
1282
|
+
|
|
1283
|
+
try:
|
|
1284
|
+
while not stop_flag.is_set():
|
|
1285
|
+
lines = []
|
|
1286
|
+
for gpu, handle in handles.items():
|
|
1287
|
+
mem = pynvml.nvmlDeviceGetMemoryInfo(handle)
|
|
1288
|
+
used = mem.used / 1024**3
|
|
1289
|
+
total = mem.total / 1024**3
|
|
1290
|
+
pct = (used / total) * 100
|
|
1291
|
+
lines.append(f"GPU {gpu}: {used:.1f}/{total:.1f} GiB ({pct:.0f}%)")
|
|
1292
|
+
|
|
1293
|
+
logger.info(" | ".join(lines))
|
|
1294
|
+
time.sleep(60)
|
|
1295
|
+
|
|
1296
|
+
finally:
|
|
1297
|
+
pynvml.nvmlShutdown()
|
|
1298
|
+
|
|
1299
|
+
|
|
1300
|
+
@staticmethod
|
|
1301
|
+
def kill_hint_processes(proc) :
|
|
1302
|
+
"""
|
|
1303
|
+
Kill all processes related to the current Python executable, typically used to terminate any remaining AlphaFold jobs after a KeyboardInterrupt (Ctrl+C).
|
|
1304
|
+
|
|
1305
|
+
Parameters :
|
|
1306
|
+
----------
|
|
1307
|
+
proc : subprocess obj
|
|
1308
|
+
"""
|
|
1309
|
+
logger.info("Ctrl+C detected")
|
|
1310
|
+
try :
|
|
1311
|
+
os.killpg(os.getpgid(proc.pid), signal.SIGKILL) #kill all PID
|
|
1312
|
+
except ProcessLookupError :
|
|
1313
|
+
pass
|