Fast-HInt-ppi 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1313 @@
1
+ import os
2
+ import pickle
3
+ import numpy as np
4
+ import matplotlib.pyplot as plt
5
+ import csv
6
+ import logging
7
+ import subprocess
8
+ import multiprocessing
9
+ import pynvml
10
+ import copy
11
+ import signal
12
+ import glob
13
+ import math
14
+ import random
15
+ import json
16
+ import string
17
+ import time
18
+
19
+ from pathlib import Path
20
+ from Bio import SeqIO
21
+ from concurrent.futures import ThreadPoolExecutor
22
+ from .Scoring_HInt import Score_interaction
23
+
24
+
25
+ # Configure global logger
26
+ logging.basicConfig(
27
+ filename="./log_file/HInt.log", # Log file name
28
+ level=logging.INFO, # Log level
29
+ format="%(asctime)s - %(levelname)s - %(message)s" # Log format
30
+ )
31
+
32
+ # Use the logger configuration from HInt.py
33
+ logger = logging.getLogger()
34
+
35
+
36
+
37
+ def Define_informations() :
38
+ """
39
+ Parse the HInt.txt configuration file and collect all user-defined parameters into a single dictionary.
40
+
41
+ - Reads key : value pairs from HInt.txt (comments starting with # are ignored)
42
+ - Verifies the presence of mandatory parameters
43
+ - Assigns default values to optional or empty parameters
44
+ - Normalizes paths (removes trailing '/')
45
+ - Parses interaction definitions (including multimers and interaction regions)
46
+ - Validates biological constraints (DeepLoc, homo-oligomer, number of baits)
47
+
48
+ Returns:
49
+ -------
50
+ Informations_dict : dict
51
+ Dictionary containing all validated and formatted configuration parameters required by HInt.
52
+
53
+ """
54
+ logger.info("Defining informations")
55
+ Informations_dict = dict()
56
+ list_inf = ["Signal_peptide", "Homo-oligomer", "Interact_with", "Organism", "DeepLoc", "Regions", "Multimer_bait", "AlphaFold", "Max_protein_lenght", "Min_protein_lenght", "Path_Uniprot_ID", "Path_AlphaFold_Data", "Path_Pickle_Feature", "Path_Singularity_Image", "Path_MMseqs2_Data"]
57
+ with open("HInt.txt", "r") as file :
58
+ for lines in file :
59
+ if ":" in lines :
60
+ info = lines.split("#")[0]
61
+ informations_name = info.split(":")[0].strip().strip("\n")
62
+ informations = info.split(":")[1].strip().strip("\n")
63
+ Informations_dict[informations_name] = informations
64
+ for info in list_inf :
65
+ if info not in Informations_dict.keys() : #if settings file is not authentic
66
+ if info in ["Interact_with","Path_Uniprot_ID", "Path_AlphaFold_Data", "Path_Pickle_Feature"] :
67
+ raise ValueError(f"HInt.txt file is compromised, verify the file. {info} is missing")
68
+ elif info in ["Signal_peptide","Homo-oligomer","Path_MMseqs2_Data","Regions","Multimer_bait","DeepLoc","AlphaFold","Max_protein_lenght","Min_protein_lenght","Organism"] :
69
+ Informations_dict[info] = ""
70
+
71
+ ### Normalize all configuration values
72
+ for informations_key in Informations_dict.keys() : #verify all informations and set default value
73
+ if type(Informations_dict[informations_key]) is str and Informations_dict[informations_key].endswith("/") : #avoid error in path
74
+ Informations_dict[informations_key] = Informations_dict[informations_key][:-1]
75
+ if len(Informations_dict[informations_key]) == 0 :
76
+ logger.info(f'{informations_key} is empty')
77
+ if informations_key == "Path_ccp4" :
78
+ if os.path.isdir("/opt/xtal/ccp4-9") == True :
79
+ logger.info("Set ccp4 path by default on /opt/xtal/ccp4-9")
80
+ Informations_dict[informations_key] = "/opt/xtal/ccp4-9"
81
+ else :
82
+ raise ValueError("Path_ccp4 is empty and ccp4 is not found in /opt/xtal/ccp4-9. You need to install ccp4 or set the path of ccp4 in HInt.txt")
83
+ elif informations_key == "Path_AlphaFold_Data" :
84
+ if os.path.isdir("./alphadata") == True :
85
+ logger.info("Set AlphaFold data by default on ./alphadata")
86
+ Informations_dict[informations_key] = "./alphadata"
87
+ else :
88
+ raise ValueError("Path_AlphaFold_Data is empty and AlphaFold data is not found in ./alphadata. You need to download AlphaFold data or set the path of AlphaFold data in HInt.txt")
89
+ elif informations_key == "Path_Pickle_Feature" :
90
+ logger.info("Set pickle feature path by default on ./feature")
91
+ Informations_dict[informations_key] = "./feature"
92
+ elif informations_key == "Path_MMseqs2_Data" :
93
+ logger.info("/!\ local MMseqs2 GPU will not be used")
94
+ elif informations_key == "Signal_peptide" :
95
+ Informations_dict[informations_key] = "None"
96
+ elif informations_key == "Homo-oligomer" :
97
+ Informations_dict[informations_key] = "1"
98
+ elif informations_key == "DeepLoc" :
99
+ Informations_dict[informations_key] = "None"
100
+ elif informations_key == "AlphaFold" :
101
+ Informations_dict[informations_key] = "2"
102
+ logger.info("Set AlphaFold version by default on AlphaFold2")
103
+ elif informations_key == "Min_protein_lenght" :
104
+ Informations_dict[informations_key] = "20"
105
+ logger.info("Minimum lenght for prey protein set by default 20 to AA")
106
+ elif informations_key == "Max_protein_lenght" :
107
+ Informations_dict[informations_key] = ""
108
+ elif informations_key == "Organism" :
109
+ Informations_dict[informations_key] = "None"
110
+ if len(Informations_dict[informations_key]) != 0 :
111
+ if informations_key == "Path_AlphaFold_Data" :
112
+ if os.path.isdir(Informations_dict[informations_key]) == False :
113
+ raise ValueError(f"Path set for AlphaFold database doesn't exist : {Informations_dict[informations_key]}")
114
+ elif informations_key == "Path_MMseqs2_Data" :
115
+ if os.path.isdir(Informations_dict[informations_key]) == False :
116
+ raise ValueError (f"Path set for MMseq database doesn't exist : {Informations_dict[informations_key]}")
117
+ elif informations_key == "Path_ccp4" :
118
+ if os.path.isdir(Informations_dict[informations_key]) == False :
119
+ raise ValueError (f"Path set for ccp4 doesn't exist : {Informations_dict[informations_key]}")
120
+ ### Parse protein-protein interaction definitions
121
+ # Multiple baits
122
+ # Multimeric baits
123
+ # Region-specific interactions
124
+ if informations_key == "Interact_with" :
125
+ regions_dict = dict()
126
+ new_baits_list = list()
127
+ if "[" in Informations_dict["Interact_with"] : #create multimer for bait
128
+ list_complex = [prot.strip("[").strip(",").strip() for prot in Informations_dict["Interact_with"].split("]")]
129
+ if "" in list_complex :
130
+ list_complex.remove("")
131
+ nbr_baits = len(list_complex)
132
+ list_baits = [prot.strip("[").strip("]") for prot in Informations_dict["Interact_with"].split(",")]
133
+ Informations_dict["Multimer_bait"] = list_complex
134
+ else :
135
+ list_baits = [prot.strip(",").strip() for prot in Informations_dict["Interact_with"].split(",")]
136
+ nbr_baits = len(list_baits)
137
+ Informations_dict["Multimer_bait"] = list_baits
138
+ if nbr_baits > 3 :
139
+ raise ValueError("HInt don't support more than 3 differents baits")
140
+ for prot in list_baits :
141
+ if "-" in prot and "(" in prot and ")" in prot : #if region is specified for the bait :
142
+ name_prot = prot.split("(")[0]
143
+ new_baits_list.append(name_prot.strip())
144
+ regions_dict[name_prot.strip()] = prot.split("(")[1].strip(")")
145
+ for i,multimer in enumerate(Informations_dict["Multimer_bait"]) :
146
+ if prot in multimer :
147
+ new_multimer = multimer.replace(prot,name_prot.strip())
148
+ Informations_dict["Multimer_bait"][i] = new_multimer
149
+ else :
150
+ if prot.strip() in regions_dict.keys() and regions_dict[prot.strip()] != "0-0" :
151
+ raise ValueError(f"HInt don't support multiple regions for protein bait : {prot.strip()}")
152
+ regions_dict[prot.strip()] = "0-0"
153
+ new_baits_list.append(prot.strip())
154
+ Informations_dict["Regions"] = regions_dict
155
+ Informations_dict["Interact_with"] = new_baits_list
156
+
157
+ ### Homo-oligomer check
158
+ if informations_key == "Homo-oligomer" :
159
+ if int(Informations_dict[informations_key]) == False :
160
+ raise ValueError(f"Homo-oligomer is not an integer")
161
+ if Informations_dict[informations_key] == "0" :
162
+ Informations_dict[informations_key] = "1"
163
+ ### Organism check
164
+ if informations_key == "Organism" :
165
+ if Informations_dict[informations_key] not in ["arch", "gram+", "gram-", "euk", "None"] :
166
+ raise ValueError(f"Incorrect Organism value : {Informations_dict[informations_key]}")
167
+
168
+ ### DeepLoc check
169
+ if informations_key == "DeepLoc" :
170
+ if Informations_dict["Organism"] == "euk" : #euk
171
+ for value in Informations_dict[informations_key].split(","):
172
+ if value.strip() not in ["Cytoplasm", "Nucleus", "Extracellular", "Cell membrane", "Mitochondrion", "Plastid", "Endoplasmic reticulum", "Lysosome/Vacuole", "Golgo apparatus", "Peroxisome", "None"] :
173
+ raise ValueError(f"Incorrect DeepLoc value : {value}")
174
+ else : #other
175
+ for value in Informations_dict[informations_key].split(","):
176
+ if value.strip() not in ["Cell wall & surface", "Extracellular", "Cytoplasmic", "Cytoplasmic Membrane", "Outer Membrane", "Periplasmic", "None"] :
177
+ raise ValueError(f"Incorrect DeepLocPro value : {value}")
178
+
179
+ if len(Informations_dict["Signal_peptide"]) == 0 and len(Informations_dict["DeepLoc"]) == 0 and len(Informations_dict["Homo-oligomer"]) == 0 and len(Informations_dict["Interact_with"]) == 0 : #no info
180
+ logger.info("Need information to discriminate the potential homolog/interolog")
181
+ exit()
182
+ return(Informations_dict)
183
+
184
+ def run_deeploc(file, org, need_DeepLoc, GPU) :
185
+ """
186
+ Run DeepLoc (DeepLoc2 or DeepLocPro) to predict the subcellular localization of proteins requiring DeepLoc annotation.
187
+
188
+ - Extracts protein sequences (with signal peptide handling) from a File_proteins object
189
+ - Writes a temporary FASTA file containing only proteins needing DeepLoc prediction
190
+ - Runs the appropriate DeepLoc software depending on the organism type
191
+ - Parses the output CSV file
192
+ - Stores predicted localizations in the result dictionary
193
+
194
+ Parameters :
195
+ ----------
196
+ file : object of class File_proteins
197
+ org : str
198
+ need_DeepLoc : list
199
+ GPU : list
200
+
201
+ """
202
+ result_dict = file.get_result_dict()
203
+ prot_seq = file.get_proteins_sequence_SP()
204
+ deeploc = file.get_deeploc()
205
+ GPU_str = ""
206
+ for nbr_GPU in GPU :
207
+ GPU_str += nbr_GPU + ","
208
+ GPU_str = GPU_str.strip(",")
209
+ file_name = file.get_file_name().split("/")[-1]
210
+ ext_f = "." + file_name.split(".")[-1]
211
+ fasta_file = file_name.replace(ext_f,"_msa.fasta")
212
+
213
+
214
+ if os.path.exists("log_file/result_deeploc") == True :
215
+ os.system("rm -r log_file/result_deeploc")
216
+ dp_lines = str()
217
+ for protein in need_DeepLoc :
218
+ dp_lines += ">"+protein+"\n"+prot_seq[protein]+"\n"
219
+ with open(f"log_file/{fasta_file}", "w") as dp_file :
220
+ dp_file.write(dp_lines)
221
+
222
+ if org == "euk" :
223
+ logger.info("Start DeepLoc eucaryote")
224
+ software = "deeploc2"
225
+ cmd = f"CUDA_VISIBLE_DEVICES={GPU_str} {software} -f log_file/{fasta_file} -o log_file/result_deeploc -d cuda"
226
+ else :
227
+ logger.info("Start DeepLocPro")
228
+ software = "deeplocpro"
229
+ if org == "gram-" :
230
+ group = "negative"
231
+ if org == "gram+" :
232
+ group = "positive"
233
+ if org == "arch" :
234
+ group = "archaea"
235
+ cmd = f"CUDA_VISIBLE_DEVICES={GPU_str} {software} -f log_file/{fasta_file} -o log_file/result_deeploc -d cuda -g {group}" #deeplocpro don't use all GPU
236
+ os.system(cmd)
237
+ DeepLoc_file = os.listdir(f"log_file/result_deeploc")
238
+ if software == "deeplocpro" :
239
+ min_index = 3
240
+ min_score = 0.1
241
+ index_name = 1
242
+ else :
243
+ min_index = 4
244
+ min_score = 0.4
245
+ index_name = 0
246
+ with open(f"log_file/result_deeploc/{DeepLoc_file[0]}", "r") as DL_file :
247
+ reader = csv.reader(DL_file, delimiter=',')
248
+ for index, line in enumerate(reader) :
249
+ compartment = str()
250
+ if index == 0 :
251
+ first_line = line #save title name
252
+ else :
253
+ protein = line[index_name]
254
+ for index, score in enumerate(line) :
255
+ if index >= min_index and float(score) > min_score :
256
+ compartment += first_line[index] + "|"
257
+ compartment = compartment.strip("|")
258
+
259
+ deeploc[protein] = compartment
260
+ result_dict[protein]["DeepLoc"] = deeploc[protein]
261
+ file.set_result_dict(result_dict)
262
+ file.set_deeploc(deeploc)
263
+
264
+
265
+
266
+ def run_SP (file, Informations_dict, need_SP, need_msa) :
267
+ """
268
+ Remove signal peptides from protein sequences using SignalP and generate new FASTA sequences without signal peptides.
269
+
270
+ - Runs SignalP on proteins requiring signal peptide (SP) detection
271
+ - Extracts cleavage positions from SignalP output
272
+ - Generates new sequences without signal peptides
273
+ - Updates internal sequence storage
274
+ - Returns the list of proteins still requiring MSA computation
275
+ - Updates the result dictionary with signal peptide presence/absence
276
+
277
+ If a protein already has an MSA but does not yet have a sequence without signal peptide, it will be processed here but not returned for MSA.
278
+
279
+ Parameters :
280
+ ----------
281
+ file : object of class File_proteins
282
+ Informations_dict : dict
283
+ need_SP : list
284
+ List of protein identifiers requiring signal peptide detection/removal.
285
+ need_msa : list
286
+ List of protein identifiers requiring MSA generation.
287
+
288
+ Returns :
289
+ ----------
290
+ new_need_msa : list
291
+ """
292
+ final_file = str()
293
+ SP_signal = 0
294
+ file_name = file.get_file_name().split("/")[-1]
295
+ ext_f = "." + file_name.split(".")[-1]
296
+ fasta_file = file_name.replace(ext_f,"_msa.fasta")
297
+ output_file = fasta_file.replace(".fasta","")
298
+ new_fasta_dict = file.get_proteins_sequence_no_SP()
299
+ result_dict = file.get_result_dict()
300
+ if_prot_SP = file.get_prot_SP()
301
+ prot_SP_len = dict()
302
+
303
+ file.create_fasta_file(True, need_SP)
304
+ cmd = f"signalp -fasta log_file/{fasta_file} -org {Informations_dict['Organism']} -prefix log_file/{output_file}"
305
+ os.system(cmd)
306
+
307
+ file_signalp = fasta_file.replace(".fasta","_summary.signalp5")
308
+ with open(f"log_file/{file_signalp}","r") as fh :
309
+ for line in fh :
310
+ new_line = line.split("\t")
311
+ if new_line[1] != "OTHER" and new_line[0][0] != "#" :
312
+ prot_SP_len[new_line[0]] = new_line[len(new_line)-1].split("-")[1].split(".")[0]
313
+ result_dict[new_line[0]]["Signal_peptide"] = "Yes"
314
+ if_prot_SP[new_line[0]] = "Yes"
315
+ elif new_line[1] == "OTHER" and new_line[0][0] != "#" :
316
+ result_dict[new_line[0]]["Signal_peptide"] = "No"
317
+ if_prot_SP[new_line[0]] = "No"
318
+ with open(f"log_file/{fasta_file}", "r") as fa_file :
319
+ for line2 in fa_file :
320
+ new_line2 = line2
321
+ if SP_signal == 0 and line2[0] != ">" :
322
+ new_line2 = line2
323
+ new_fasta_dict[save_key] = line2.strip("\n")
324
+ if int(SP_signal) > 0 :
325
+ new_line2 = line2[int(SP_signal)-1:len(line2)]
326
+ new_fasta_dict[save_key] = line2[int(SP_signal)-1:len(line2)].strip("\n")
327
+ SP_signal = 0
328
+ if line2[0] == ">" :
329
+ save_key = line2[1:len(line2)-1]
330
+ if str(line2[1:len(line2)-1]) in prot_SP_len.keys() :
331
+ SP_signal = prot_SP_len[line2[1:len(line2)-1]]
332
+ final_file = final_file + new_line2
333
+
334
+ file.set_proteins_sequence_no_SP(new_fasta_dict)
335
+ file.set_result_dict(result_dict)
336
+ file.set_prot_SP(if_prot_SP)
337
+ return need_msa
338
+
339
+ def check_exist_MSA(file, Informations_dict, need_msa) :
340
+ """
341
+ Check for the existence of precomputed MSAs.
342
+ - Searches for precomputed MSA
343
+ - Updates the list of proteins requiring MSA generation based on existing files
344
+
345
+ Parameters :
346
+ ----------
347
+ file : object of class File_proteins
348
+ Informations_dict : dict
349
+ need_msa : list
350
+
351
+ Returns :
352
+ ----------
353
+ new_need_msa : list
354
+ List of protein identifiers still requiring MSA generation after checking for existing files.
355
+ """
356
+ new_fasta_dict = file.get_proteins_sequence_no_SP()
357
+ new_need_msa = list()
358
+ for prot in need_msa :
359
+ if os.path.isfile(f"{Informations_dict['Path_Pickle_Feature']}/{prot}.a3m") == False :
360
+ new_need_msa.append(prot)
361
+ if os.path.isfile(f"{Informations_dict['Path_Pickle_Feature']}/{prot}.a3m") == True : #if MSA already exist but have SP
362
+ with open(f"{Informations_dict['Path_Pickle_Feature']}/{prot}.a3m", "r") as msa_file :
363
+ b = "no"
364
+ for line in msa_file :
365
+ if b == "yes" :
366
+ first_line = line.strip("\n")
367
+ break
368
+ if line[0] == ">" :
369
+ b = "yes"
370
+ if first_line != new_fasta_dict[prot] : #if sequence with SP is different from sequence without SP, modify the MSA file with the new sequence without SP
371
+ cmd_rm = f"rm -r {Informations_dict['Path_Pickle_Feature']}/{prot}*"
372
+ os.system(cmd_rm)
373
+ new_need_msa.append(prot)
374
+ return new_need_msa
375
+
376
+ def create_feature (file, Informations_dict, GPU, CPU, need_msa, need_pkl) :
377
+ """
378
+ Generate MSAs and AlphaFold feature pickle files for a set of proteins.
379
+
380
+ - Searches for precomputed MSAs in the AlphaFold database
381
+ - Generates MSAs using ColabFold/MMseqs2 when missing
382
+ - Uses CPU parallelization (mafft) and GPU acceleration (MMseqs2 GPU)
383
+ - Updates the lists of proteins requiring MSA or pickle generation
384
+
385
+ Parameters :
386
+ ----------
387
+ file : object of class File_proteins
388
+ informations_dict : dict
389
+ GPU : list
390
+ CPU : int
391
+ need_msa : list
392
+ need_pkl : list
393
+
394
+ """
395
+ if need_msa == [] and need_pkl == [] :
396
+ return
397
+ AF_version = Informations_dict["AlphaFold"]
398
+ Path_AlphaFold_Data = Informations_dict["Path_AlphaFold_Data"]
399
+ Path_Pickle_Feature = Informations_dict["Path_Pickle_Feature"]
400
+ uniprot_prot = file.get_uniprot_prot()
401
+ prot_no_SP = file.get_proteins_sequence_no_SP()
402
+ prot_SP = file.get_proteins_sequence_SP()
403
+ file_name = file.get_file_name()
404
+ ext_f = "." + file_name.split(".")[-1]
405
+ msa_name = file_name.replace(ext_f,"_msa.fasta")
406
+ retry_queue = list()
407
+ GPU_str = str()
408
+ for nbr_GPU in GPU :
409
+ GPU_str += nbr_GPU + ","
410
+ GPU_str = GPU_str.strip(",")
411
+ #logger.info(f"GPU use : {GPU_str}")
412
+ logger.info(f"{len(need_msa)} proteins need MSA")
413
+ logger.info(f"{len(need_pkl)} proteins need pkl files")
414
+ if os.path.exists(Path_Pickle_Feature) == False :
415
+ os.system(f"mkdir {Path_Pickle_Feature}")
416
+
417
+ def AFDB_msa_search(Path_Pickle_Feature,prot_SP,prot_no_SP,generated_msa) :
418
+ retry_queue = list() #list of protein to retry if error during MSA search in AFdb (too many request on server)
419
+ with ThreadPoolExecutor(max_workers=max_workers_mafft) as executor :
420
+ for protein in generated_msa : #check if prot have an MSA in alphafold database
421
+ l_p = len(protein)
422
+ if l_p >= 5 and l_p <= 10 and "_" not in protein and protein in uniprot_prot : #if not, is not an UniprotID #avoid name with UniprotID but different sequence
423
+ on_afdb = True
424
+ time.sleep(0.2) #avoid too many request on AFdb server
425
+ url = f"https://alphafold.ebi.ac.uk/files/msa/AF-{protein}-F1-msa_v6.a3m" #do it linearly because of error with parallelization, due to too many request on AFdb server
426
+ msa_in = f"{Path_Pickle_Feature}/{protein}.a3m"
427
+ result = subprocess.run(["wget", "-O", msa_in, url], stderr=subprocess.PIPE)
428
+ if result.returncode != 0 :
429
+ err = result.stderr.decode()
430
+ on_afdb = False
431
+ if "404" in err : #404 error mean that MSA is not available in AFdb
432
+ os.remove(msa_in)
433
+ if "429" in err or "Too Many Requests" in err :
434
+ retry_queue.append(protein)
435
+
436
+ else :
437
+ nbr_line = sum(1 for _ in open(msa_in))
438
+ if nbr_line < 3 : #if MSA have only 1 sequence, not useful for AF2 prediction, so generated it with mmseqs2
439
+ on_afdb = False
440
+ os.remove(msa_in)
441
+ if on_afdb == True :
442
+ SP = len(prot_SP[protein]) - len(prot_no_SP[protein])
443
+ if SP > 0 : #if no SP don't modify the MSA
444
+ executor.submit(fetch_trim_mafft, protein, Path_Pickle_Feature, prot_SP, prot_no_SP)
445
+ logger.info(f"MSA for {protein} processed")
446
+ need_msa.remove(protein) #msa found
447
+ need_pkl.append(protein)
448
+ return retry_queue
449
+
450
+ start = time.time()
451
+ generated_msa = copy.deepcopy(need_msa)
452
+ if need_msa != [] :
453
+ #ThreadPool to parallelise Mafft
454
+ max_workers_mafft = CPU # how many jobs in parallel
455
+
456
+ #Look for MSA in AlphaFold database
457
+ logger.info(f"Search MSA in AlphaFold database")
458
+
459
+ retry_queue = AFDB_msa_search(Path_Pickle_Feature,prot_SP,prot_no_SP,generated_msa)
460
+
461
+ while retry_queue != [] : #retry if error during MSA search in AFdb
462
+ retry_queue = AFDB_msa_search(Path_Pickle_Feature,prot_SP,prot_no_SP,retry_queue)
463
+
464
+ logger.info("MSA search in AlphaFold database complete")
465
+ file.create_fasta_file(False, need_msa, need_pkl)
466
+
467
+ #Create MSA files with ColabFold mmseq2 classic pipeline for proteins without MSA, and pickle files for proteins generated with MMseqs2 GPU
468
+ if AF_version == "2" : # Cause AF3 if too fast for hmmer MSA generation, so we use mmseqs2 classic pipeline for AF3
469
+ mmseqs2 = "True"
470
+ else :
471
+ mmseqs2 = "False"
472
+ if len(need_msa) > 0 :
473
+ max_workers_feature = CPU
474
+ futures_list = []
475
+ with ThreadPoolExecutor(max_workers=max_workers_feature) as executor : #CPU parallelization
476
+ for protein in need_msa :
477
+ msa_name = f"./log_file/{protein}_msa.fasta"
478
+ cmd = ["create_individual_features.py",
479
+ f"--fasta_paths={msa_name}",
480
+ f"--data_dir=/data/alphadata_v2",
481
+ "--save_msa_files=True",
482
+ f"--output_dir={Path_Pickle_Feature}",
483
+ "--max_template_date=2024-05-02",
484
+ "--skip_existing=True",
485
+ f"--use_mmseqs2={mmseqs2}",
486
+ "--use_precomputed_msas=True"]
487
+ futures_list.append(executor.submit(create_ind_feature, protein, msa_name, prot_no_SP, cmd))
488
+ end = time.time()
489
+ elapsed = end - start
490
+ logger.info("Create MSA take "+ str(elapsed/60)+" minutes")
491
+
492
+ #Create pkl files for proteins without pkl file (from MSA found in AFdb or error during MSA generation with ColabFold mmseqs2)
493
+ if len(need_pkl) > 0 :
494
+ max_workers_feature = CPU
495
+ futures_list = []
496
+
497
+ with ThreadPoolExecutor(max_workers=max_workers_feature) as executor : #CPU parallelization
498
+ for protein in need_pkl :
499
+ pkl_file = f"./log_file/{protein}_pkl.fasta"
500
+ cmd2 = ["create_individual_features.py",
501
+ f"--fasta_paths={pkl_file}",
502
+ f"--data_dir={Path_AlphaFold_Data}",
503
+ "--save_msa_files=True",
504
+ f"--output_dir={Path_Pickle_Feature}",
505
+ "--max_template_date=2024-05-02",
506
+ "--skip_existing=True",
507
+ "--use_mmseqs2=True",
508
+ "--use_precomputed_msas=True",
509
+ "--re_search_templates_mmseqs2=True"]
510
+ futures_list.append(executor.submit(create_ind_feature, protein, pkl_file, prot_no_SP, cmd2))
511
+
512
+ def create_ind_feature(protein, pkl_file, prot_no_SP, cmd) :
513
+ """
514
+ Create a pickle file for a single protein using the create_individual_features.py script. Allow parallelization.
515
+
516
+ Parameters :
517
+ ----------
518
+ protein : str
519
+ pkl_file : str
520
+ prot_no_SP : dict
521
+ cmd : str
522
+ """
523
+ with open(pkl_file, "w") as pkl_f :
524
+ pkl_f.write(f">{protein}\n{prot_no_SP[protein]}\n")
525
+ process = subprocess.Popen(cmd, stderr=subprocess.STDOUT,stdout=subprocess.PIPE, text=True, bufsize=1, universal_newlines=True)
526
+ for line in process.stdout :
527
+ print(f"{line}", end="") # stream live
528
+ os.remove(pkl_file)
529
+ process.wait()
530
+
531
+ def fetch_trim_mafft(protein, Path_Pickle_Feature, prot_SP, prot_no_SP) :
532
+ """
533
+ Trim signal peptides if present, and realign the MSA using MAFFT.
534
+
535
+ - Removes signal peptide residues from all sequences when necessary
536
+ - Filters out excessively short or invalid sequences after trimming
537
+ - Realigns the trimmed MSA using MAFFT
538
+ - Reformats the alignment back to A3M format
539
+
540
+ Parameters :
541
+ ----------
542
+ protein : str
543
+ Path_Pickle_Feature : str
544
+ prot_SP : dict
545
+ prot_no_SP : dict
546
+ """
547
+ msa_in = f"{Path_Pickle_Feature}/{protein}.a3m"
548
+ aln_out = f"{Path_Pickle_Feature}/{protein}.aln"
549
+
550
+ SP = len(prot_SP[protein]) - len(prot_no_SP[protein])
551
+ trimmed_records = []
552
+ for rec in SeqIO.parse(msa_in, "fasta") :
553
+ new_seq = rec.seq[SP:]
554
+ len_aa_count = sum(1 for c in new_seq if c.isupper())
555
+ if len_aa_count < 10 or not any(c.isupper() for c in str(new_seq)) : #skip short sequence after SP cut or sequence don't have aa
556
+ continue
557
+ new_rec = rec[:]
558
+ new_rec.seq = new_seq
559
+ trimmed_records.append(new_rec)
560
+
561
+ if trimmed_records :
562
+ with open(msa_in, "w") as msa_file :
563
+ for rec in trimmed_records:
564
+ msa_file.write(f">{rec.description}\n{rec.seq}\n")
565
+
566
+ #realign with mafft
567
+ cmd_mafft = f"mafft --quiet --anysymbol --thread 1 --parttree --retree 1 --maxiterate 0 {msa_in} > {aln_out}" #monothread
568
+ subprocess.run(cmd_mafft, shell=True, check=True)
569
+
570
+ #reformat a3m
571
+ cmd_reformat = f"reformat.pl fas a3m {aln_out} {msa_in}"
572
+ subprocess.run(cmd_reformat, shell=True, check=True)
573
+ cmd_rm_f = f"rm {aln_out}"
574
+ subprocess.run(cmd_rm_f, shell=True, check=True)
575
+
576
+ #delete \n
577
+ all_lines = ""
578
+ with open(msa_in, "r") as in_a3m:
579
+ seq, header = "", None
580
+ for line in in_a3m:
581
+ if line.startswith(">"):
582
+ if header:
583
+ all_lines += f"{header}\n{seq}\n"
584
+ header = line.strip()
585
+ seq = ""
586
+ else:
587
+ seq += line.strip()
588
+ if header:
589
+ all_lines += f"{header}\n{seq}\n"
590
+ with open(msa_in, "w") as out_a3m:
591
+ out_a3m.write(all_lines)
592
+
593
+
594
+
595
+ def filter_signalP(file, Informations_dict, need_msa, need_pkl) :
596
+ """
597
+ Filter proteins based on the presence of a signal peptide using SignalP results.
598
+
599
+ - Checks the first amino acid of each protein sequence to determine if a signal peptide is present (assumes sequences without SP start with 'M')
600
+ - Updates the result dictionary with "Signal_peptide" status
601
+ - Filters proteins from MSA and feature generation lists based on the user-specified SignalP
602
+ - Keeps all proteins if no filtering criterion is specified
603
+
604
+ Parameters :
605
+ ----------
606
+ file : object of class File_proteins
607
+ Informations_dict : dict
608
+ need_msa : list
609
+ need_pkl : list
610
+
611
+ Returns :
612
+ ----------
613
+ need_msa : list
614
+ need_pkl : list
615
+ """
616
+ result_dict = file.get_result_dict()
617
+ possible_prey = file.get_possible_prey()
618
+ new_possible_prey = list()
619
+ SignalP = Informations_dict["Signal_peptide"]
620
+ for protein in possible_prey :
621
+ if SignalP == "None" :
622
+ new_possible_prey.append(protein)
623
+ elif SignalP == result_dict[protein]["Signal_peptide"] :
624
+ new_possible_prey.append(protein)
625
+ elif SignalP != result_dict[protein]["Signal_peptide"] :
626
+ result_dict[protein]["Reason_for_filtering"] = "Signal peptide : SignalP"
627
+ if protein in need_msa :
628
+ need_msa.remove(protein)
629
+ if protein in need_pkl :
630
+ need_pkl.remove(protein)
631
+ if SignalP != "None" :
632
+ logger.info("Protein preys remaining after SignalP filtering : " + str(len(new_possible_prey)))
633
+ file.set_possible_prey(new_possible_prey)
634
+ file.set_result_dict(result_dict)
635
+ return need_msa, need_pkl
636
+
637
+ def Make_all_MSA_coverage(file, Path_Pickle_Feature, baits, prey) :
638
+ """
639
+ Generate MSA coverage plots for all proteins and create a shallow_MSA summary file.
640
+
641
+ - Computes sequence coverage and identity from the MSA
642
+ - Generates a coverage heatmap saved as PDF
643
+ - Determines if the MSA is "shallow" or missing (based on number of sequences)
644
+ - Updates the result dictionary and possible prey list accordingly
645
+ - Writes a summary of shallow MSAs to "log_file/shallow_MSA.txt"
646
+
647
+ Parameters :
648
+ ----------
649
+ file : object of class File_proteins
650
+ Path_Pickle_Feature : string
651
+ baits : list
652
+ prey : list
653
+ """
654
+ shallow_MSA = str()
655
+ result_dict = file.get_result_dict()
656
+ all_prot = prey
657
+ if baits != [''] :
658
+ for bait in baits :
659
+ all_prot.append(bait)
660
+ new_possible_prey = list()
661
+ for prot in all_prot :
662
+ if Path(f'{Path_Pickle_Feature}/{prot}_coverage.pdf').exists() == False :
663
+ #Make coverage plot
664
+ pre_feature_dict = pickle.load(open(f"{Path_Pickle_Feature}/{prot}.pkl","rb"))
665
+ feature_dict = pre_feature_dict.feature_dict
666
+ msa = feature_dict['msa']
667
+ seqid = (np.array(msa[0] == msa).mean(-1))
668
+ seqid_sort = seqid.argsort()
669
+ non_gaps = (msa != 21).astype(float)
670
+ non_gaps[non_gaps == 0] = np.nan
671
+ final = non_gaps[seqid_sort] * seqid[seqid_sort, None]
672
+ plt.figure(figsize=(14, 4), dpi=100)
673
+ plt.subplot(1, 2, 1)
674
+ plt.title(f"Sequence coverage ({prot})")
675
+ plt.imshow(final, interpolation="nearest", aspect="auto", cmap="rainbow_r", vmin=0, vmax=1, origin="lower")
676
+ plt.plot((msa != 21).sum(0), color="black")
677
+ plt.xlim(-0.5, msa.shape[1] - 0.5)
678
+ plt.ylim(-0.5, msa.shape[0] - 0.5)
679
+ plt.colorbar(label="Sequence identity to query", )
680
+ plt.xlabel("Positions")
681
+ plt.ylabel("Sequences")
682
+ plt.savefig(f"{Path_Pickle_Feature}/{prot+('_' if prot else '')}coverage.pdf")
683
+ plt.close()
684
+ #Evaluate MSA depth
685
+ with open(f'{Path_Pickle_Feature}/{prot}.a3m') as msa_f :
686
+ line_msa = sum(1 for _ in msa_f)
687
+ real_nb_msa = line_msa/2
688
+ if real_nb_msa <= 100 and prot not in baits :
689
+ if real_nb_msa < 2 :
690
+ shallow_MSA += prot + " : " + str(int(real_nb_msa)) + " sequences\n"
691
+ result_dict[prot]["shallow_MSA"] = "No MSA"
692
+ result_dict[prot]["Reason_for_filtering"] = "No MSA"
693
+ else :
694
+ shallow_MSA += prot + " : " + str(int(real_nb_msa)) + " sequences\n"
695
+ result_dict[prot]["shallow_MSA"] = "Shallow MSA"
696
+ new_possible_prey.append(prot)
697
+ if real_nb_msa > 100 and prot not in baits :
698
+ new_possible_prey.append(prot)
699
+ if real_nb_msa <= 100 and prot in baits :
700
+ logger.warning(f"This bait : {prot} have a shallow MSA with {int(real_nb_msa)} sequences. Predicted interactions involving this protein should be interpreted with caution.")
701
+ with open("log_file/shallow_MSA.txt", "w") as MSA_file :
702
+ MSA_file.write(shallow_MSA)
703
+ file.set_possible_prey(new_possible_prey)
704
+ file.set_result_dict(result_dict)
705
+
706
+
707
+
708
+ def filter_lenght(file, Informations_dict, need_msa, need_pkl, need_DeepLoc) :
709
+ """
710
+ Filter proteins based on their sequence length and update the list of possible preys.
711
+
712
+ - Uses the full sequence (including signal peptide) to evaluate protein length
713
+ - Filters proteins that are either too short or too long according to user-defined thresholds
714
+ - Updates the result dictionary with a reason for filtering
715
+ - Removes filtered proteins from MSA, pickle, and DeepLoc processing lists
716
+ - Updates the File_proteins object with the new prey list
717
+
718
+ Parameters :
719
+ ----------
720
+ file : object of class File_proteins
721
+ Informations_dict : dict
722
+ need_msa : list
723
+ need_pkl : list
724
+ need_DeepLoc : list
725
+
726
+ Returns :
727
+ ----------
728
+ need_msa : list
729
+ need_pkl : list
730
+ need_DeepLoc : list
731
+ """
732
+ result_dict = file.get_result_dict()
733
+ sequence_dict = file.get_proteins_sequence_SP() #use sequence with SP for lenght filtering
734
+ possible_prey = file.get_possible_prey()
735
+ if Informations_dict["Max_protein_lenght"] == "" : #not set by default, depend of GPU memory
736
+ max_lenght = 100000
737
+ else :
738
+ max_lenght = int(Informations_dict["Max_protein_lenght"])
739
+ min_lenght = int(Informations_dict["Min_protein_lenght"]) #set by default to 20
740
+ new_possible_prey = list()
741
+ for protein in possible_prey :
742
+ if len(sequence_dict[protein]) < max_lenght and len(sequence_dict[protein]) > min_lenght :
743
+ new_possible_prey.append(protein)
744
+ else :
745
+ result_dict[protein]["Reason_for_filtering"] = "Lenght filtering"
746
+ if protein in need_msa :
747
+ need_msa.remove(protein)
748
+ if protein in need_pkl :
749
+ need_pkl.remove(protein)
750
+ if protein in need_DeepLoc :
751
+ need_DeepLoc.remove(protein)
752
+ logger.info("Protein preys remaining after lenght filtering : " + str(len(new_possible_prey)))
753
+ file.set_possible_prey(new_possible_prey)
754
+ file.set_result_dict(result_dict)
755
+ return(need_msa, need_pkl, need_DeepLoc)
756
+
757
+
758
+ def filter_deeploc(file, Informations_dict, need_msa, need_pkl) :
759
+ """
760
+ Filter proteins based on predicted cellular localization and update the prey list as well as the MSA and pickle processing lists.
761
+
762
+ - Compares each protein's predicted DeepLoc localization(s) with the user-specified allowed localizations
763
+ - Keeps proteins matching at least one allowed localization
764
+ - Marks proteins that do not match as filtered in the result dictionary
765
+ - Updates the need_msa and need_pkl lists to include only proteins that pass the filter or are baits
766
+
767
+ Parameters :
768
+ ----------
769
+ file : object of class File_proteins
770
+ Informations_dict : dict
771
+ need_msa : list
772
+ need_pkl : list
773
+
774
+ Returns :
775
+ ----------
776
+ new_need_msa : list
777
+ new_need_pkl : list
778
+ """
779
+ localisation = Informations_dict["DeepLoc"].split(",")
780
+ localisation = [loc.strip() for loc in localisation]
781
+ deeploc = file.get_deeploc()
782
+ possible_baits = Informations_dict["Interact_with"]
783
+ possible_prey = file.get_possible_prey()
784
+ result_dict = file.get_result_dict()
785
+ new_possible_prey = list()
786
+ new_need_msa = list()
787
+ new_need_pkl = list()
788
+ for protein in possible_prey :
789
+ for loc in deeploc[protein].split("|") :
790
+ if loc in localisation :
791
+ new_possible_prey.append(protein)
792
+ break #stop the loop if one localisation is correct
793
+ if protein not in new_possible_prey :
794
+ result_dict[protein]["Reason_for_filtering"] = "Cellular localisation : DeepLoc"
795
+ for prot_msa in need_msa :
796
+ if prot_msa in new_possible_prey or prot_msa in possible_baits :
797
+ new_need_msa.append(prot_msa)
798
+ for prot_pkl in need_pkl :
799
+ if prot_pkl in new_possible_prey or prot_pkl in possible_baits :
800
+ new_need_pkl.append(prot_pkl)
801
+ logger.info("Protein prey remaining after DeepLoc filtering : " + str(len(new_possible_prey)))
802
+ file.set_possible_prey(new_possible_prey)
803
+ file.set_result_dict(result_dict)
804
+ return(new_need_msa, new_need_pkl)
805
+
806
+
807
+ def Generate_scripts(file, Informations_dict, Interaction_file, bait) :
808
+ """
809
+ Prepare the list of interaction jobs, taking into account protein lengths and GPU VRAM constraints to avoid out-of-memory (OOM) errors.
810
+
811
+ - Estimates the maximum number of amino acids allowed based on available GPU VRAM
812
+ - Builds interaction jobs (PPI or homo-oligomeric) only if they fit in memory
813
+ - Skips interactions already computed
814
+ - Records interactions that are too large for the GPU in a log file
815
+
816
+ Parameters :
817
+ ----------
818
+ file : object of class File_proteins
819
+ Informations_dict : dict
820
+ Interaction_file : str
821
+ bait : str
822
+
823
+ Returns :
824
+ ----------
825
+ job_with_vram_length : list of tuple
826
+ """
827
+ job_with_vram_length = [] # [(job_str, int_length)]
828
+ OOM_int = ""
829
+ result_dict = file.get_result_dict()
830
+ AF_version = Informations_dict["AlphaFold"]
831
+ possible_prey = file.get_possible_prey()
832
+ lenght_prot = file.get_lenght_prot()
833
+ regions = Informations_dict["Regions"]
834
+
835
+ # Estimate max amino acids based on GPU VRAM (choose first GPU as reference)
836
+ pynvml.nvmlInit()
837
+ handle = pynvml.nvmlDeviceGetHandleByIndex(0)
838
+ mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
839
+ vram = (mem_info.total / 1024**2) * 0.001 # GiB
840
+ max_aa = int((0.0000627 + math.sqrt(0.0000627**2 - 4*0.00000332*(3.8 - vram))) / (2*0.00000332))
841
+ pynvml.nvmlShutdown()
842
+
843
+ if Interaction_file == "PPI_int" or Interaction_file == "Compounds" :
844
+ # Determine bait length and region
845
+ complexe = "," in bait
846
+ if complexe :
847
+ save_multimer = bait
848
+ bait_file = save_multimer.replace(",", "_and_")
849
+ bait_for_job = save_multimer.replace(",", ";")
850
+ lenght = sum(lenght_prot[prot] for prot in save_multimer.split(","))
851
+ for prot in save_multimer.split(",") :
852
+ if regions[prot] != "0-0":
853
+ start, end = int(regions[prot].split("-")[0]), int(regions[prot].split("-")[1])
854
+ bait_file = bait_file.replace(prot, f"{prot}_{start}-{end}")
855
+ bait_for_job = bait_for_job.replace(prot, f"{prot},{start}-{end}")
856
+ else :
857
+ lenght = lenght_prot[bait]
858
+ if regions[bait] != "0-0" :
859
+ start, end = int(regions[bait].split("-")[0]), int(regions[bait].split("-")[1])
860
+ bait_file = f"{bait}_{start}-{end}"
861
+ bait_for_job = f"{bait},{start}-{end}"
862
+ else :
863
+ bait_file = bait
864
+ bait_for_job = bait
865
+
866
+ # Build job list
867
+ if Interaction_file == "PPI_int" :
868
+ copy_possible_prey = copy.deepcopy(possible_prey) # To avoid modifying the list while iterating
869
+ for prey in copy_possible_prey :
870
+ int_lenght = lenght + lenght_prot[prey]
871
+
872
+ # Check if model already exists
873
+ if AF_version == "3" :
874
+ path1 = glob.glob(f"./result_PPI_int/{bait_file}_and_{prey}/*_model.cif")
875
+ path2 = glob.glob(f"./result_PPI_int/{prey}_and_{bait_file}/*_model.cif")
876
+ else : # AF_version == "2"
877
+ path1 = glob.glob(f"./result_PPI_int/{bait_file}_and_{prey}/ranked_0.pdb")
878
+ path2 = glob.glob(f"./result_PPI_int/{prey}_and_{bait_file}/ranked_0.pdb")
879
+
880
+ if len(path1) == 0 and len(path2) == 0 :
881
+ if int_lenght <= max_aa :
882
+ job_str = f"{bait_for_job};{prey}\n"
883
+ vram_lenght = 3.8 + (-0.0000627) * int_lenght + 0.00000332 * int_lenght**2
884
+ job_with_vram_length.append((job_str, vram_lenght))
885
+ else :
886
+ OOM_int += f"{bait_for_job};{prey}\n"
887
+ result_dict[prey][f"Reason_for_filtering"] = "Interaction too large for your GPU, possible prey"
888
+ possible_prey.remove(prey)
889
+ if Interaction_file == "Compounds" :
890
+ Compounds = file.get_compounds()
891
+ vram_lenght = 1.9 + (-0.0000627) * lenght + 0.00000332 * lenght**2
892
+ for compound in Compounds.keys() :
893
+ job_str = f"{bait_for_job};{compound}\n"
894
+ path = glob.glob(f"./result_Compounds/{bait_file}_and_{compound}/*_model.cif")
895
+ if len(path) == 0 :
896
+ if os.path.isdir(f"./result_Compounds/{bait_file}_and_{compound}") : #if dir exist and model not exist, rm dir
897
+ os.system(f"rm -r ./result_Compounds/{bait_file}_and_{compound}")
898
+ job_with_vram_length.append((f"{job_str}", vram_lenght)) #consider no vram for compound, only for protein bait
899
+
900
+ if Interaction_file == "homo_int" :
901
+ nbr_oligo = Informations_dict.get("Homo-oligomer", 2)
902
+ for prey in possible_prey :
903
+ int_lenght = lenght_prot[prey] * int(nbr_oligo)
904
+ if AF_version == "3" :
905
+ path = glob.glob(f"./result_homo_int/{prey}_homo_{nbr_oligo}er/*_model.cif")
906
+ else :
907
+ path = glob.glob(f"./result_homo_int/{prey}_homo_{nbr_oligo}er/ranked_0.pdb")
908
+ if len(path) == 0 :
909
+ if int_lenght <= max_aa :
910
+ if AF_version == "3" :
911
+ job_str = f"{prey}_af3_input.json:{nbr_oligo}\n"
912
+ if AF_version == "2" :
913
+ job_str = f"{prey}:{nbr_oligo}\n"
914
+ vram_lenght = 3.8 + (-0.0000627) * int_lenght + 0.00000332 * int_lenght**2
915
+ job_with_vram_length.append((job_str, vram_lenght))
916
+ else :
917
+ OOM_int += f"{prey}:{nbr_oligo}\n"
918
+ result_dict[prey]["Reason_for_filtering"] = "Homo-oligomer too large for your GPU"
919
+ possible_prey.remove(prey)
920
+
921
+
922
+ # Classify job_list in function of int lenght
923
+ job_with_vram_length.sort(key=lambda x: x[1], reverse=True)
924
+
925
+
926
+ # Save OOM interactions
927
+ with open("log_file/OOM_interactions.txt", "w") as OOM_file :
928
+ OOM_file.write(OOM_int)
929
+
930
+ file.set_possible_prey(possible_prey)
931
+ file.set_result_dict(result_dict)
932
+
933
+ return job_with_vram_length
934
+
935
+
936
+ def Generate_first_batch(job_with_vram_length, GPU, multi_job_per_gpu) :
937
+ if not job_with_vram_length :
938
+ return
939
+
940
+ pynvml.nvmlInit()
941
+ handle = pynvml.nvmlDeviceGetHandleByIndex(int(GPU[0]))
942
+ mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
943
+ max_vram = mem_info.total / 1024**3 #Gb
944
+ pynvml.nvmlShutdown()
945
+
946
+ gpu_vram_used = {gpu: 0.0 for gpu in GPU}
947
+ gpu_jobs = []
948
+
949
+ multi_job_per_gpu = False if multi_job_per_gpu == "False" else True
950
+
951
+ pending = list(job_with_vram_length)
952
+
953
+ for interaction, job_vram in pending :
954
+
955
+ sorted_gpus = sorted(GPU, key=lambda g: gpu_vram_used[g])
956
+
957
+ launched = False
958
+
959
+ for gpu_id in sorted_gpus :
960
+
961
+ free = max_vram - gpu_vram_used[gpu_id]
962
+
963
+ if job_vram <= free and (multi_job_per_gpu or len(gpu_jobs[gpu_id]) == 0) :
964
+
965
+ gpu_jobs.append(interaction.split(";")[-1].strip("\n"))
966
+ gpu_vram_used[gpu_id] += job_vram
967
+
968
+ launched = True
969
+ break
970
+
971
+ if not launched :
972
+ continue
973
+
974
+ return gpu_jobs, gpu_vram_used
975
+
976
+ def prioritize_by_vram_fit(jobs, GPU, max_per_batch=3):
977
+
978
+ import pynvml
979
+
980
+ pynvml.nvmlInit()
981
+ handle = pynvml.nvmlDeviceGetHandleByIndex(int(GPU[0]))
982
+ max_vram = pynvml.nvmlDeviceGetMemoryInfo(handle).total / 1024**3
983
+ pynvml.nvmlShutdown()
984
+
985
+ jobs = sorted(jobs, key=lambda x: x[1], reverse=True)
986
+
987
+ remaining = jobs.copy()
988
+ all_batches = []
989
+
990
+ while remaining:
991
+
992
+ batch = []
993
+ used_vram = 0.0
994
+
995
+ # on tente plusieurs passes pour remplir correctement
996
+ changed = True
997
+
998
+ while changed and len(batch) < max_per_batch:
999
+ changed = False
1000
+
1001
+ for i in range(len(remaining)):
1002
+
1003
+ job, vram = remaining[i]
1004
+
1005
+ if len(batch) < max_per_batch and used_vram + vram <= max_vram:
1006
+ batch.append(job.split(";")[-1].strip("\n"))
1007
+ used_vram += vram
1008
+ remaining.pop(i)
1009
+ changed = True
1010
+ break
1011
+
1012
+ all_batches.append(batch)
1013
+
1014
+ return all_batches
1015
+
1016
+ def Generate_3D_model(HInt_object, CPU, multi_scoring, Informations_dict, interaction_type, job_with_vram_length, GPU, multi_job_per_gpu, seq_bait={}, compounds_dict={}) :
1017
+ """
1018
+ Generate 3D models using multiple GPUs and multiprocessing.
1019
+
1020
+ Parameters :
1021
+ ----------
1022
+ Informations_dict : dict
1023
+ interaction_files : str
1024
+ job_with_vram_length : list of tuple
1025
+ GPU : list
1026
+ multi_job_per_gpu : str
1027
+ seq_bait : dict
1028
+ compounds_dict : dict
1029
+ """
1030
+ if not job_with_vram_length :
1031
+ return
1032
+
1033
+ pynvml.nvmlInit()
1034
+ handle = pynvml.nvmlDeviceGetHandleByIndex(int(GPU[0]))
1035
+ mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
1036
+ max_vram = mem_info.total / 1024**3 #Gb
1037
+ pynvml.nvmlShutdown()
1038
+
1039
+ stop_flag = multiprocessing.Event()
1040
+ monitor = multiprocessing.Process(
1041
+ target=monitor_vram,
1042
+ args=(GPU, stop_flag)
1043
+ )
1044
+ monitor.start()
1045
+
1046
+ with open("log_file/All_PPI_jobs.txt", "w") as f : #write complete script for informations
1047
+ for job, _ in job_with_vram_length :
1048
+ f.write(job)
1049
+ manager(jobs_pending=list(job_with_vram_length),
1050
+ HInt_object=HInt_object,
1051
+ CPU=CPU,
1052
+ multi_scoring=multi_scoring,
1053
+ Informations_dict=Informations_dict,
1054
+ GPU=GPU,
1055
+ max_vram=max_vram,
1056
+ interaction_type=interaction_type,
1057
+ multi_job_per_gpu=multi_job_per_gpu,
1058
+ seq_bait=seq_bait,
1059
+ compounds_dict=compounds_dict
1060
+ )
1061
+
1062
+ stop_flag.set()
1063
+ monitor.join()
1064
+
1065
+ def manager(jobs_pending, HInt_object, CPU, multi_scoring, Informations_dict, GPU, max_vram, interaction_type, multi_job_per_gpu, seq_bait, compounds_dict) :
1066
+ """
1067
+ Manage repartition of jobs on GPUs, monitor VRAM usage, and handle job completion.
1068
+
1069
+ Parameters :
1070
+ ----------
1071
+ jobs_pending : list
1072
+ GPU : list
1073
+ max_vram : float
1074
+ Path_AlphaFold_Data : str
1075
+ Path_Pickle_Feature : str
1076
+ interaction_type : str
1077
+ AF_version : str
1078
+ multi_job_per_gpu : str
1079
+ seq_bait : dict
1080
+ Baits : list
1081
+ compounds_dict : dict
1082
+ """
1083
+ AF_version = Informations_dict["AlphaFold"]
1084
+ Path_AlphaFold_Data = Informations_dict["Path_AlphaFold_Data"]
1085
+ Path_Pickle_Feature = Informations_dict["Path_Pickle_Feature"]
1086
+ Baits = Informations_dict["Interact_with"]
1087
+ compound = None
1088
+ manager_proc = multiprocessing.Manager()
1089
+ result_queue = manager_proc.Queue()
1090
+ multi_job_per_gpu = False if multi_job_per_gpu == "False" else True
1091
+ gpu_vram_used = {gpu: 0.0 for gpu in GPU}
1092
+ jobs_running = {gpu: [] for gpu in GPU} # {gpu_id: {job_str: process}}
1093
+
1094
+ while jobs_pending or any(jobs_running.values()):
1095
+
1096
+ while not result_queue.empty() :
1097
+ status, job, gpu_id, vram = result_queue.get()[:4]
1098
+ gpu_vram_used[gpu_id] -= vram
1099
+ for i, (j, p) in enumerate(jobs_running[gpu_id]):
1100
+ if j == job:
1101
+ p.join()
1102
+ jobs_running[gpu_id].pop(i)
1103
+ Score_interaction(HInt_object, Informations_dict, CPU, interaction_type, job, multi_scoring, "")
1104
+ HInt_object.Make_save_dict()
1105
+ break
1106
+
1107
+ launched = False
1108
+
1109
+ for interaction, job_vram in list(jobs_pending) :
1110
+ sorted_gpus = sorted(GPU, key=lambda g: gpu_vram_used[g]) #make PPI on GPU with minimum vram use
1111
+ if interaction_type == "Compounds" :
1112
+ compound = compounds_dict[interaction.strip("\n").split(";")[-1]] #to just select one compound for each interaction
1113
+ for gpu_id in sorted_gpus :
1114
+
1115
+ free = max_vram - gpu_vram_used[gpu_id]
1116
+ if job_vram <= free and (multi_job_per_gpu or len(jobs_running[gpu_id]) == 0) :
1117
+ p = multiprocessing.Process(target=gpu_job_runner,
1118
+ args=(gpu_id,
1119
+ interaction,
1120
+ job_vram,
1121
+ result_queue,
1122
+ Path_AlphaFold_Data,
1123
+ Path_Pickle_Feature,
1124
+ interaction_type,
1125
+ AF_version,
1126
+ seq_bait,
1127
+ Baits,
1128
+ compound
1129
+ ),
1130
+ )
1131
+
1132
+ p.start()
1133
+
1134
+ gpu_vram_used[gpu_id] += job_vram
1135
+ jobs_running[gpu_id].append((interaction, p))
1136
+ jobs_pending.remove((interaction, job_vram))
1137
+ launched = True
1138
+ break
1139
+
1140
+ if launched :
1141
+ break
1142
+ if not launched :
1143
+ time.sleep(1)
1144
+
1145
+
1146
+ def gpu_job_runner(gpu_id, interaction_file, vram, result_queue, Path_AlphaFold_Data, Path_Pickle_Feature, interaction_type, AF_version, seq_bait, Baits, compound) :
1147
+ """
1148
+ Run a single AlphaFold job on a specified GPU, monitor its completion, and report results.
1149
+
1150
+ Parameters :
1151
+ ----------
1152
+ gpu_id : int
1153
+ interaction_file : str
1154
+ vram : float
1155
+ result_queue : multiprocessing.Queue
1156
+ Path_AlphaFold_Data : str
1157
+ Path_Pickle_Feature : str
1158
+ interaction_type : str
1159
+ AF_version : str
1160
+ seq_bait : dict
1161
+ Baits : list
1162
+ compound : str
1163
+ """
1164
+ env = os.environ.copy()
1165
+ env['CUDA_VISIBLE_DEVICES'] = str(gpu_id)
1166
+ env['XLA_PYTHON_CLIENT_PREALLOCATE'] = 'false'
1167
+ env['TF_FORCE_UNIFIED_MEMORY'] = 'true'
1168
+ env['XLA_FLAGS'] = '--xla_gpu_enable_triton_gemm=false'
1169
+
1170
+ if interaction_type == "PPI_int" or interaction_type == "Compounds" :
1171
+ prot_int = interaction_file.strip("\n").replace(";", "_and_")
1172
+ if interaction_type == "homo_int" :
1173
+ prot_int = interaction_file.strip("\n").replace(":", "_homo_")
1174
+ prot_int += "er"
1175
+ prot_int = prot_int.replace(",","_")
1176
+
1177
+ file_name = f"{interaction_type}_GPU_{prot_int}.txt"
1178
+
1179
+ if interaction_type == "Compounds" : #Write json file to use AF3 native
1180
+ random_seed = random.randrange(2**32 - 1)
1181
+ sequences = []
1182
+ ligands = []
1183
+ i = -1
1184
+ for obj in interaction_file.strip("\n").split(";") :
1185
+ i += 1
1186
+ if "," in obj :
1187
+ obj = obj.split(",")[0]
1188
+ if obj in Baits : #it's protein
1189
+ with open(f"{Path_Pickle_Feature}/{obj}.a3m", "r") as a3m_f :
1190
+ all_lines = a3m_f.read()
1191
+ sequences.append({
1192
+ "protein": {
1193
+ "id": string.ascii_uppercase[i],
1194
+ "sequence": seq_bait[obj],
1195
+ "unpairedMsa": all_lines,
1196
+ "pairedMsa": all_lines,
1197
+ "templates": []
1198
+ }
1199
+ })
1200
+
1201
+ else : # it's LIGAND (SMILES)
1202
+ ligands.append({
1203
+ "ligand": {
1204
+ "id": string.ascii_uppercase[i],
1205
+ "smiles": compound
1206
+ }
1207
+ })
1208
+ af3_input = {
1209
+ "name": prot_int,
1210
+ "dialect": "alphafold3",
1211
+ "version": 3,
1212
+ "modelSeeds": [random_seed],
1213
+ "sequences": sequences + ligands
1214
+ }
1215
+ file_name = file_name.replace(".txt", ".json")
1216
+ with open(f"log_file/{file_name}", "w") as f:
1217
+ json.dump(af3_input, f, indent=2)
1218
+ try :
1219
+ if interaction_type != "Compounds" :
1220
+ with open(f"log_file/{file_name}", "w") as f :
1221
+ f.write(interaction_file)
1222
+
1223
+ if AF_version == "2" :
1224
+ cmd = (
1225
+ "run_multimer_jobs.py --mode=custom "
1226
+ "--num_cycle=3 "
1227
+ "--num_predictions_per_model=1 "
1228
+ "--compress_result_pickles=True "
1229
+ f"--output_path=./result_{interaction_type} "
1230
+ f"--data_dir={Path_AlphaFold_Data} "
1231
+ f"--protein_lists=log_file/{file_name} "
1232
+ f"--monomer_objects_dir={Path_Pickle_Feature} "
1233
+ "--remove_keys_from_pickles=False"
1234
+ )
1235
+ if AF_version == "3" and interaction_type != "Compounds" :
1236
+ cmd = (
1237
+ "run_multimer_jobs.py --mode=custom "
1238
+ f"--output_path=./result_{interaction_type} "
1239
+ f"--data_dir={Path_AlphaFold_Data} "
1240
+ f"--protein_lists=log_file/{file_name} "
1241
+ f"--monomer_objects_dir={Path_Pickle_Feature} "
1242
+ "--fold_backend=alphafold3 "
1243
+ "--use_ap_style=True "
1244
+ )
1245
+
1246
+ if AF_version == "3" and interaction_type == "Compounds" :
1247
+ cmd = (f"python ./alphafold3/run_alphafold.py --output_dir=./result_{interaction_type} --db_dir={Path_AlphaFold_Data} --model_dir={Path_AlphaFold_Data} --json_path=log_file/{file_name}")
1248
+ logger.info(f"[GPU {gpu_id}] Starting {interaction_file}")
1249
+ proc = subprocess.Popen(
1250
+ cmd,
1251
+ shell=True,
1252
+ env=env,
1253
+ preexec_fn=os.setsid
1254
+ )
1255
+
1256
+ while True :
1257
+
1258
+ ret = proc.poll()
1259
+ if ret is not None:
1260
+ if ret != 0 :
1261
+ raise RuntimeError(f"Crash ({ret})")
1262
+ break
1263
+
1264
+ time.sleep(0.5)
1265
+
1266
+ result_queue.put(("done", interaction_file, gpu_id, vram))
1267
+ os.system(f"rm log_file/{file_name}")
1268
+ logger.info(f"[GPU {gpu_id}] Finished {interaction_file}")
1269
+
1270
+ except KeyboardInterrupt :
1271
+ kill_hint_processes(proc)
1272
+
1273
+
1274
+
1275
+ def monitor_vram(GPU, stop_flag) :
1276
+ """
1277
+ Monitor VRAM usage for given GPUs every `interval` seconds.
1278
+ """
1279
+
1280
+ pynvml.nvmlInit()
1281
+ handles = {int(gpu): pynvml.nvmlDeviceGetHandleByIndex(int(gpu)) for gpu in GPU}
1282
+
1283
+ try:
1284
+ while not stop_flag.is_set():
1285
+ lines = []
1286
+ for gpu, handle in handles.items():
1287
+ mem = pynvml.nvmlDeviceGetMemoryInfo(handle)
1288
+ used = mem.used / 1024**3
1289
+ total = mem.total / 1024**3
1290
+ pct = (used / total) * 100
1291
+ lines.append(f"GPU {gpu}: {used:.1f}/{total:.1f} GiB ({pct:.0f}%)")
1292
+
1293
+ logger.info(" | ".join(lines))
1294
+ time.sleep(60)
1295
+
1296
+ finally:
1297
+ pynvml.nvmlShutdown()
1298
+
1299
+
1300
+ @staticmethod
1301
+ def kill_hint_processes(proc) :
1302
+ """
1303
+ Kill all processes related to the current Python executable, typically used to terminate any remaining AlphaFold jobs after a KeyboardInterrupt (Ctrl+C).
1304
+
1305
+ Parameters :
1306
+ ----------
1307
+ proc : subprocess obj
1308
+ """
1309
+ logger.info("Ctrl+C detected")
1310
+ try :
1311
+ os.killpg(os.getpgid(proc.pid), signal.SIGKILL) #kill all PID
1312
+ except ProcessLookupError :
1313
+ pass