PyamilySeq 0.9.0__py3-none-any.whl → 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- PyamilySeq-1.0.0.dist-info/METADATA +17 -0
- PyamilySeq-1.0.0.dist-info/RECORD +6 -0
- {PyamilySeq-0.9.0.dist-info → PyamilySeq-1.0.0.dist-info}/WHEEL +1 -1
- PyamilySeq-1.0.0.dist-info/entry_points.txt +2 -0
- PyamilySeq-1.0.0.dist-info/top_level.txt +1 -0
- PyamilySeq/Cluster_Summary.py +0 -163
- PyamilySeq/Constants.py +0 -2
- PyamilySeq/Group_Splitter.py +0 -382
- PyamilySeq/PyamilySeq.py +0 -296
- PyamilySeq/PyamilySeq_Genus.py +0 -242
- PyamilySeq/PyamilySeq_Species.py +0 -287
- PyamilySeq/Seq_Combiner.py +0 -67
- PyamilySeq/__init__.py +0 -0
- PyamilySeq/clusterings.py +0 -362
- PyamilySeq/utils.py +0 -408
- PyamilySeq-0.9.0.dist-info/METADATA +0 -345
- PyamilySeq-0.9.0.dist-info/RECORD +0 -16
- PyamilySeq-0.9.0.dist-info/entry_points.txt +0 -5
- PyamilySeq-0.9.0.dist-info/top_level.txt +0 -1
- {PyamilySeq-0.9.0.dist-info → PyamilySeq-1.0.0.dist-info}/LICENSE +0 -0
PyamilySeq/PyamilySeq.py
DELETED
|
@@ -1,296 +0,0 @@
|
|
|
1
|
-
import argparse
|
|
2
|
-
import collections
|
|
3
|
-
import os
|
|
4
|
-
import glob
|
|
5
|
-
import subprocess
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
try:
|
|
10
|
-
from .PyamilySeq_Species import cluster as species_cluster
|
|
11
|
-
from .PyamilySeq_Genus import cluster as genus_cluster
|
|
12
|
-
from .Constants import *
|
|
13
|
-
from .utils import *
|
|
14
|
-
except (ModuleNotFoundError, ImportError, NameError, TypeError) as error:
|
|
15
|
-
from PyamilySeq_Species import cluster as species_cluster
|
|
16
|
-
from PyamilySeq_Genus import cluster as genus_cluster
|
|
17
|
-
from Constants import *
|
|
18
|
-
from utils import *
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
def run_cd_hit(options, input_file, clustering_output, clustering_mode):
|
|
24
|
-
cdhit_command = [
|
|
25
|
-
clustering_mode,
|
|
26
|
-
'-i', input_file,
|
|
27
|
-
'-o', clustering_output,
|
|
28
|
-
'-c', str(options.pident),
|
|
29
|
-
'-s', str(options.len_diff),
|
|
30
|
-
'-T', str(options.threads),
|
|
31
|
-
'-M', str(options.clustering_memory),
|
|
32
|
-
'-d', "0",
|
|
33
|
-
'-g', "1",
|
|
34
|
-
'-sc', "1",
|
|
35
|
-
'-sf', "1"
|
|
36
|
-
]
|
|
37
|
-
if options.verbose != None:
|
|
38
|
-
subprocess.run(cdhit_command)
|
|
39
|
-
else:
|
|
40
|
-
subprocess.run(cdhit_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def main():
|
|
44
|
-
parser = argparse.ArgumentParser(description='PyamilySeq ' + PyamilySeq_Version + ': A tool that groups genes into unique clusters.')
|
|
45
|
-
### Required Arguments
|
|
46
|
-
required = parser.add_argument_group('Required Parameters')
|
|
47
|
-
required.add_argument('-run_mode', action='store', dest='run_mode', choices=['Full','Partial'],
|
|
48
|
-
help='Run Mode: Should PyamilySeq be run in "Full" or "Partial" mode?',
|
|
49
|
-
required=True)
|
|
50
|
-
required.add_argument('-group_mode', action='store', dest='group_type', choices=['Species', 'Genus'],
|
|
51
|
-
help='Group Mode: Should PyamilySeq be run in "Species" or "Genus" mode? ',
|
|
52
|
-
required=True)
|
|
53
|
-
required.add_argument("-clustering_format", action="store", dest="clustering_format", choices=['CD-HIT','TSV','CSV'],
|
|
54
|
-
help="Clustering format to use: CD-HIT or TSV (MMseqs2, BLAST, DIAMOND) / CSV edge-list file (Node1\tNode2).",
|
|
55
|
-
required=True)
|
|
56
|
-
required.add_argument("-output_dir", action="store", dest="output_dir",
|
|
57
|
-
help="Directory for all output files.",
|
|
58
|
-
required=True)
|
|
59
|
-
### Full-Mode Arguments
|
|
60
|
-
full_mode_args = parser.add_argument_group('Full-Mode Parameters - Required when "-run_mode Full" is used')
|
|
61
|
-
full_mode_args.add_argument("-input_type", action="store", dest="input_type", choices=['separate', 'combined'],
|
|
62
|
-
help="Type of input files: 'separate' for separate FASTA and GFF files,"
|
|
63
|
-
" 'combined' for GFF files with embedded FASTA sequences.",
|
|
64
|
-
required=False)
|
|
65
|
-
full_mode_args.add_argument("-input_dir", action="store", dest="input_dir",
|
|
66
|
-
help="Directory containing GFF/FASTA files.",
|
|
67
|
-
required=False)
|
|
68
|
-
full_mode_args.add_argument("-name_split", action="store", dest="name_split",
|
|
69
|
-
help="substring used to split the filename and extract the genome name ('_combined.gff3' or '.gff').",
|
|
70
|
-
required=False)
|
|
71
|
-
full_mode_args.add_argument('-sequence_type', action='store', dest='sequence_type', default='DNA',choices=['AA', 'DNA'],
|
|
72
|
-
help='Default - DNA: Should clustering be performed in "DNA" or "AA" mode?',
|
|
73
|
-
required=False)
|
|
74
|
-
full_mode_args.add_argument('-gene_ident', action='store', dest='gene_ident', default='CDS',
|
|
75
|
-
help='Identifier used for extraction of sequences such as "misc_RNA,gene,mRNA,CDS,rRNA,tRNA,tmRNA,CRISPR,ncRNA,regulatory_region,oriC,pseudo"',
|
|
76
|
-
required=False)
|
|
77
|
-
full_mode_args.add_argument("-pid", action="store", dest="pident", type=float, default=0.95,
|
|
78
|
-
help="Default 0.95: Pident threshold for clustering.",
|
|
79
|
-
required=False)
|
|
80
|
-
full_mode_args.add_argument("-len_diff", action="store", dest="len_diff", type=float, default=0.80,
|
|
81
|
-
help="Default 0.80: Minimum length difference between clustered sequences - (-s) threshold for CD-HIT clustering.",
|
|
82
|
-
required=False)
|
|
83
|
-
###Clustering Arguments
|
|
84
|
-
clustering_args = parser.add_argument_group('Clustering Runtime Arguments - Optional when "-run_mode Full" is used')
|
|
85
|
-
clustering_args.add_argument("-mem", action="store", dest="clustering_memory", type=int, default=4000,
|
|
86
|
-
help="Default 4000: Memory to be allocated for clustering (in MBs).",
|
|
87
|
-
required=False)
|
|
88
|
-
clustering_args.add_argument("-t", action="store", dest="threads", type=int, default=8,
|
|
89
|
-
help="Default 8: Threads to be allocated for clustering and/or alignment.",
|
|
90
|
-
required=False)
|
|
91
|
-
|
|
92
|
-
###Partial-Mode Arguments
|
|
93
|
-
partial_mode_args = parser.add_argument_group("Partial-Mode Parameters - Required when '-run_mode Partial' is used")
|
|
94
|
-
partial_mode_args.add_argument("-cluster_file", action="store", dest="cluster_file",
|
|
95
|
-
help="Clustering output file containing CD-HIT, TSV or CSV Edge List",
|
|
96
|
-
required=False)
|
|
97
|
-
|
|
98
|
-
###Grouping Arguments
|
|
99
|
-
grouping_args = parser.add_argument_group('Grouping Parameters - Use to fine-tune grouping of genes after clustering')
|
|
100
|
-
grouping_args.add_argument('-reclustered', action='store', dest='reclustered',
|
|
101
|
-
help='Currently only works on Partial Mode: Clustering output file from secondary round of clustering.',
|
|
102
|
-
required=False)
|
|
103
|
-
grouping_args.add_argument('-seq_tag', action='store', dest='sequence_tag', default='StORF',
|
|
104
|
-
help='Default - "StORF": Unique identifier to be used to distinguish the second of two rounds of clustered sequences',
|
|
105
|
-
required=False)
|
|
106
|
-
grouping_args.add_argument('-core_groups', action="store", dest='core_groups', default="99,95,15",
|
|
107
|
-
help='Default - (\'99,95,15\'): Gene family groups to use for "Species" mode',
|
|
108
|
-
required=False)
|
|
109
|
-
|
|
110
|
-
grouping_args.add_argument('-genus_groups', action="store", dest='genus_groups', default="1,2,3,4,5,6",
|
|
111
|
-
help='Default - (\'1,2,3,4,5,6\'): Gene family groups to use for "Genus" mode',
|
|
112
|
-
required=False)
|
|
113
|
-
|
|
114
|
-
###Output Arguments
|
|
115
|
-
output_args = parser.add_argument_group('Output Parameters')
|
|
116
|
-
output_args.add_argument('-w', action="store", dest='write_groups', default=None,
|
|
117
|
-
help='Default - No output: Output sequences of identified groups (provide levels at which to output - Species "-w 99,95" Genus "-w 2,3"'
|
|
118
|
-
' - Must provide FASTA file with -original_fasta if in Partial run mode.',
|
|
119
|
-
required=False)
|
|
120
|
-
output_args.add_argument('-a', action="store_true", dest='align_core', default=None,
|
|
121
|
-
help='Default - No output: SLOW! (Only works for Species mode) Output aligned and concatinated sequences of identified groups -'
|
|
122
|
-
'provide group levels at which to output "-w 99,95" - Must provide FASTA file with -original_fasta in Partial'
|
|
123
|
-
'run mode.',
|
|
124
|
-
required=False)
|
|
125
|
-
output_args.add_argument('-original_fasta', action='store', dest='original_fasta',
|
|
126
|
-
help='FASTA file to use in conjunction with "-w" or "-con" when running in Partial Mode.',
|
|
127
|
-
required=False)
|
|
128
|
-
output_args.add_argument('-no_gpa', action='store_false', dest='gene_presence_absence_out',
|
|
129
|
-
help='Do not create a Roary/Panaroo formatted gene_presence_absence.csv (created by default) - Required for Coinfinder and other downstream tools',
|
|
130
|
-
required=False)
|
|
131
|
-
|
|
132
|
-
### Misc Arguments
|
|
133
|
-
misc = parser.add_argument_group("Misc Parameters")
|
|
134
|
-
misc.add_argument("-verbose", action="store_true", dest="verbose",
|
|
135
|
-
help="Print verbose output.",
|
|
136
|
-
required=False)
|
|
137
|
-
misc.add_argument("-v", "--version", action="version",
|
|
138
|
-
version=f"PyamilySeq version {PyamilySeq_Version} - Exiting",
|
|
139
|
-
help="Print out version number and exit")
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
options = parser.parse_args()
|
|
143
|
-
print("Running PyamilySeq: " + PyamilySeq_Version)
|
|
144
|
-
|
|
145
|
-
### Checking all required parameters are provided by user #!!# Doesn't seem to work
|
|
146
|
-
if options.run_mode == 'Full':
|
|
147
|
-
if options.reclustered != None:
|
|
148
|
-
sys.exit("Currently reclustering only works on Partial Mode.")
|
|
149
|
-
required_full_mode = [options.input_type, options.input_dir, options.name_split, options.clustering_format,
|
|
150
|
-
options.pident, options.len_diff]
|
|
151
|
-
if all(required_full_mode):
|
|
152
|
-
# Proceed with the Full mode
|
|
153
|
-
pass
|
|
154
|
-
else:
|
|
155
|
-
missing_options = [opt for opt in
|
|
156
|
-
['input_type', 'input_dir', 'name_split', 'clustering_format', 'pident', 'len_diff'] if
|
|
157
|
-
not options.__dict__[opt]]
|
|
158
|
-
print(f"Missing required options for Full mode: {', '.join(missing_options)}")
|
|
159
|
-
if options.align_core != None:
|
|
160
|
-
if options.write_groups == None:
|
|
161
|
-
sys.exit('Must provide "-w" to output gene groups before alignment "-a" can be done.')
|
|
162
|
-
elif options.run_mode == 'Partial':
|
|
163
|
-
required_partial_mode = [options.cluster_file, ]
|
|
164
|
-
if all(required_partial_mode):
|
|
165
|
-
# Proceed with the Partial mode
|
|
166
|
-
pass
|
|
167
|
-
else:
|
|
168
|
-
missing_options = [opt for opt in
|
|
169
|
-
['cluster_file',] if
|
|
170
|
-
not options.__dict__[opt]]
|
|
171
|
-
print(f"Missing required options for Partial mode: {', '.join(missing_options)}")
|
|
172
|
-
if options.align_core != None:
|
|
173
|
-
if options.write_groups == None or options.original_fasta == None:
|
|
174
|
-
sys.exit('Must provide "-w" and "-original_fasta" to output gene groups before alignment "-a" can be done.')
|
|
175
|
-
|
|
176
|
-
if options.clustering_format == 'CD-HIT':
|
|
177
|
-
clust_affix = '.clstr'
|
|
178
|
-
elif options.clustering_format == 'TSV':
|
|
179
|
-
clust_affix = '.tsv'
|
|
180
|
-
elif options.clustering_format == 'CSV':
|
|
181
|
-
clust_affix = '.csv'
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
###External tool checks:
|
|
187
|
-
##MAFFT
|
|
188
|
-
if options.align_core == True:
|
|
189
|
-
if is_tool_installed('mafft'):
|
|
190
|
-
if options.verbose != None:
|
|
191
|
-
print("mafft is installed. Proceeding with alignment.")
|
|
192
|
-
else:
|
|
193
|
-
exit("mafft is not installed. Please install mafft to proceed.")
|
|
194
|
-
##CD-HIT
|
|
195
|
-
if options.clustering_format == 'CD-HIT' and options.run_mode == 'Full':
|
|
196
|
-
if is_tool_installed('cd-hit'):
|
|
197
|
-
if options.verbose != None:
|
|
198
|
-
print("cd-hit is installed. Proceeding with clustering.")
|
|
199
|
-
else:
|
|
200
|
-
exit("cd-hit is not installed. Please install cd-hit to proceed.")
|
|
201
|
-
|
|
202
|
-
if options.write_groups != None and options.original_fasta == False:
|
|
203
|
-
exit("-fasta must br provided if -w is used")
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
if options.cluster_file:
|
|
209
|
-
options.cluster_file = fix_path(options.cluster_file)
|
|
210
|
-
if options.reclustered:
|
|
211
|
-
options.reclustered = fix_path(options.reclustered)
|
|
212
|
-
if options.input_dir:
|
|
213
|
-
options.input_dir = fix_path(options.input_dir)
|
|
214
|
-
if options.output_dir:
|
|
215
|
-
options.output_dir = fix_path(options.output_dir)
|
|
216
|
-
|
|
217
|
-
output_path = os.path.abspath(options.output_dir)
|
|
218
|
-
combined_out_file = os.path.join(output_path, "combined_sequences.fasta")
|
|
219
|
-
clustering_output = os.path.join(output_path, 'clustering_' + options.clustering_format)
|
|
220
|
-
|
|
221
|
-
if options.group_type == 'Species':
|
|
222
|
-
options.core_groups = options.core_groups + ',0'
|
|
223
|
-
groups_to_use = options.core_groups
|
|
224
|
-
elif options.group_type == 'Genus':
|
|
225
|
-
options.genus_groups = options.genus_groups + ',>'
|
|
226
|
-
groups_to_use = options.genus_groups
|
|
227
|
-
if options.align_core != None:
|
|
228
|
-
sys.exit("-a align_core not a valid option in Genus mode.")
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
if options.run_mode == 'Full':
|
|
232
|
-
if not os.path.exists(output_path):
|
|
233
|
-
os.makedirs(output_path)
|
|
234
|
-
if options.sequence_type == 'AA':
|
|
235
|
-
clustering_mode = 'cd-hit'
|
|
236
|
-
translate = True
|
|
237
|
-
elif options.sequence_type == 'DNA':
|
|
238
|
-
clustering_mode = 'cd-hit-est'
|
|
239
|
-
translate = False
|
|
240
|
-
if options.input_type == 'separate':
|
|
241
|
-
read_separate_files(options.input_dir, options.name_split, options.gene_ident, combined_out_file, translate)
|
|
242
|
-
else:
|
|
243
|
-
read_combined_files(options.input_dir, options.name_split, options.gene_ident, combined_out_file, translate)
|
|
244
|
-
|
|
245
|
-
if options.clustering_format == 'CD-HIT':
|
|
246
|
-
run_cd_hit(options, combined_out_file, clustering_output, clustering_mode)
|
|
247
|
-
|
|
248
|
-
class clustering_options:
|
|
249
|
-
def __init__(self):
|
|
250
|
-
self.run_mode = options.run_mode
|
|
251
|
-
self.cluster_format = options.clustering_format
|
|
252
|
-
self.sequence_type = options.sequence_type
|
|
253
|
-
self.reclustered = options.reclustered
|
|
254
|
-
self.sequence_tag = options.sequence_tag
|
|
255
|
-
self.core_groups = groups_to_use
|
|
256
|
-
self.clusters = clustering_output + clust_affix
|
|
257
|
-
self.output_dir = options.output_dir
|
|
258
|
-
self.gene_presence_absence_out = options.gene_presence_absence_out
|
|
259
|
-
self.write_groups = options.write_groups
|
|
260
|
-
self.threads = options.threads
|
|
261
|
-
self.align_core = options.align_core
|
|
262
|
-
self.fasta = combined_out_file
|
|
263
|
-
self.verbose = options.verbose
|
|
264
|
-
|
|
265
|
-
clustering_options = clustering_options()
|
|
266
|
-
|
|
267
|
-
elif options.run_mode == 'Partial':
|
|
268
|
-
class clustering_options:
|
|
269
|
-
def __init__(self):
|
|
270
|
-
self.run_mode = options.run_mode
|
|
271
|
-
self.cluster_format = options.clustering_format
|
|
272
|
-
self.reclustered = options.reclustered
|
|
273
|
-
self.sequence_tag = options.sequence_tag
|
|
274
|
-
self.core_groups = groups_to_use
|
|
275
|
-
self.clusters = options.cluster_file
|
|
276
|
-
self.output_dir = options.output_dir
|
|
277
|
-
self.gene_presence_absence_out = options.gene_presence_absence_out
|
|
278
|
-
self.write_groups = options.write_groups
|
|
279
|
-
self.threads = options.threads
|
|
280
|
-
self.align_core = options.align_core
|
|
281
|
-
self.fasta = options.original_fasta
|
|
282
|
-
self.verbose = options.verbose
|
|
283
|
-
|
|
284
|
-
clustering_options = clustering_options()
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
if options.group_type == 'Species':
|
|
288
|
-
species_cluster(clustering_options)
|
|
289
|
-
elif options.group_type == 'Genus':
|
|
290
|
-
genus_cluster((clustering_options))
|
|
291
|
-
|
|
292
|
-
print("Thank you for using PyamilySeq -- A detailed user manual can be found at https://github.com/NickJD/PyamilySeq\n"
|
|
293
|
-
"Please report any issues to: https://github.com/NickJD/PyamilySeq/issues\n#####")
|
|
294
|
-
|
|
295
|
-
if __name__ == "__main__":
|
|
296
|
-
main()
|
PyamilySeq/PyamilySeq_Genus.py
DELETED
|
@@ -1,242 +0,0 @@
|
|
|
1
|
-
#from line_profiler_pycharm import profile
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
try:
|
|
5
|
-
from .Constants import *
|
|
6
|
-
from .clusterings import *
|
|
7
|
-
from .utils import *
|
|
8
|
-
except (ModuleNotFoundError, ImportError, NameError, TypeError) as error:
|
|
9
|
-
from Constants import *
|
|
10
|
-
from clusterings import *
|
|
11
|
-
from utils import *
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
def gene_presence_absence_output(options, genus_dict, pangenome_clusters_First_sorted, pangenome_clusters_First_sequences_sorted):
|
|
15
|
-
print("Outputting gene_presence_absence file")
|
|
16
|
-
output_dir = os.path.abspath(options.output_dir)
|
|
17
|
-
in_name = options.clusters.split('.')[0].split('/')[-1]
|
|
18
|
-
gpa_outfile = os.path.join(output_dir, in_name)
|
|
19
|
-
gpa_outfile = open(gpa_outfile+'_gene_presence_absence.csv','w')
|
|
20
|
-
gpa_outfile.write('"Gene","Non-unique Gene name","Annotation","No. isolates","No. sequences","Avg sequences per isolate","Genome Fragment","Order within Fragment","'
|
|
21
|
-
'"Accessory Fragment","Accessory Order with Fragment","QC","Min group size nuc","Max group size nuc","Avg group size nuc","')
|
|
22
|
-
gpa_outfile.write('","'.join(genus_dict.keys()))
|
|
23
|
-
gpa_outfile.write('"\n')
|
|
24
|
-
for cluster, sequences in pangenome_clusters_First_sequences_sorted.items():
|
|
25
|
-
average_sequences_per_genome = len(sequences) / len(pangenome_clusters_First_sorted[cluster])
|
|
26
|
-
gpa_outfile.write('"group_'+str(cluster)+'","","'+str(len(pangenome_clusters_First_sorted[cluster]))+'","'+str(len(sequences))+'","'+str(average_sequences_per_genome)+
|
|
27
|
-
'","","","","","","","","",""')
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
for genus in genus_dict.keys():
|
|
31
|
-
full_out = ''
|
|
32
|
-
tmp_list = []
|
|
33
|
-
for value in sequences:
|
|
34
|
-
if value.split('_')[0] == genus:
|
|
35
|
-
tmp_list.append(value)
|
|
36
|
-
if tmp_list:
|
|
37
|
-
full_out += ',"'+''.join(tmp_list)+'"'
|
|
38
|
-
else:
|
|
39
|
-
full_out = ',""'
|
|
40
|
-
gpa_outfile.write(full_out)
|
|
41
|
-
gpa_outfile.write('\n')
|
|
42
|
-
|
|
43
|
-
### Below is some unfinished code
|
|
44
|
-
# edge_list_outfile = open(in_name+'_edge_list.csv','w')
|
|
45
|
-
# for cluster, sequences in pangenome_clusters_First_sequences_sorted.items():
|
|
46
|
-
# output = []
|
|
47
|
-
# for entry in sequences:
|
|
48
|
-
# # Split each entry at '|'
|
|
49
|
-
# genome, gene = entry.split('|')
|
|
50
|
-
# # Format the result as "gene genome"
|
|
51
|
-
# output.append(f"{gene}\t{genome}")
|
|
52
|
-
# for line in output:
|
|
53
|
-
# edge_list_outfile.write(line + '\n')
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def get_cores(options):
|
|
59
|
-
##Calculate core groups
|
|
60
|
-
groups = OrderedDict()
|
|
61
|
-
cores = OrderedDict()
|
|
62
|
-
for group in options.core_groups.split(','):
|
|
63
|
-
first_core_group = 'First_genera_' + group
|
|
64
|
-
cores[first_core_group] = []
|
|
65
|
-
if options.reclustered != None:
|
|
66
|
-
extended_core_group = 'extended_genera_' + group
|
|
67
|
-
cores[extended_core_group] = []
|
|
68
|
-
combined_core_group = 'combined_genera_' + group
|
|
69
|
-
cores[combined_core_group] = []
|
|
70
|
-
second_core_group = 'Second_genera_' + group
|
|
71
|
-
cores[second_core_group] = []
|
|
72
|
-
only_second_core_group = 'only_Second_genera_' + group
|
|
73
|
-
cores[only_second_core_group] = []
|
|
74
|
-
return cores, groups
|
|
75
|
-
|
|
76
|
-
#@profile
|
|
77
|
-
def calc_First_only_core(cluster, First_num, cores):
|
|
78
|
-
try:
|
|
79
|
-
cores['First_genera_' + str(First_num)].append(cluster)
|
|
80
|
-
except KeyError:
|
|
81
|
-
cores['First_genera_>'].append(cluster)
|
|
82
|
-
#@profile
|
|
83
|
-
def calc_single_First_extended_Second_only_core(cluster, First_num, cores, Second_num): # Count gene families extended with StORFs
|
|
84
|
-
group = First_num + Second_num
|
|
85
|
-
try:
|
|
86
|
-
cores['extended_genera_' + str(group)].append(cluster)
|
|
87
|
-
except KeyError:
|
|
88
|
-
cores['extended_genera_>'].append(cluster)
|
|
89
|
-
#@profile
|
|
90
|
-
def calc_multi_First_extended_Second_only_core(cluster, First_num, cores, Second_num): # Count seperately those gene families extended with StORF_Reporter but combined >1 PEP
|
|
91
|
-
group = First_num + Second_num
|
|
92
|
-
try:
|
|
93
|
-
cores['combined_genera_' + str(group)].append(cluster)
|
|
94
|
-
except KeyError:
|
|
95
|
-
cores['combined_genera_>'].append(cluster)
|
|
96
|
-
#@profile
|
|
97
|
-
def calc_Second_only_core(cluster, cores, Second_num):
|
|
98
|
-
try:
|
|
99
|
-
cores['Second_genera_' + str(Second_num)].append(cluster)
|
|
100
|
-
except KeyError:
|
|
101
|
-
cores['Second_genera_>'].append(cluster)
|
|
102
|
-
#@profile
|
|
103
|
-
def calc_only_Second_only_core(cluster, cores, Second_num): # only count the true storf onlies
|
|
104
|
-
try:
|
|
105
|
-
cores['only_Second_genera_' + str(Second_num)].append(cluster)
|
|
106
|
-
except:
|
|
107
|
-
cores['only_Second_genera_>'].append(cluster)
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
#@profile
|
|
112
|
-
def cluster(options):
|
|
113
|
-
|
|
114
|
-
if options.cluster_format == 'CD-HIT':
|
|
115
|
-
genus_dict, pangenome_clusters_First, pangenome_clusters_First_genomes, pangenome_clusters_First_sequences, reps = cluster_CDHIT(options, '_')
|
|
116
|
-
elif 'TSV' in options.cluster_format or 'CSV' in options.cluster_format:
|
|
117
|
-
genus_dict, pangenome_clusters_First, pangenome_clusters_First_genomes, pangenome_clusters_First_sequences, reps = cluster_EdgeList(options, '_')
|
|
118
|
-
|
|
119
|
-
###
|
|
120
|
-
cores, groups = get_cores(options)
|
|
121
|
-
###
|
|
122
|
-
|
|
123
|
-
if options.reclustered != None:
|
|
124
|
-
if options.cluster_format == 'CD-HIT':
|
|
125
|
-
combined_pangenome_clusters_First_Second_clustered,not_Second_only_cluster_ids,combined_pangenome_clusters_Second, combined_pangenome_clusters_Second_sequences = combined_clustering_CDHIT(options, genus_dict, '_')
|
|
126
|
-
elif 'TSV' in options.cluster_format or 'CSV' in options.cluster_format:
|
|
127
|
-
combined_pangenome_clusters_First_Second_clustered,not_Second_only_cluster_ids,combined_pangenome_clusters_Second, combined_pangenome_clusters_Second_sequences = combined_clustering_Edge_List(options, '_')
|
|
128
|
-
pangenome_clusters_Type = combined_clustering_counting(options, pangenome_clusters_First, reps, combined_pangenome_clusters_First_Second_clustered, pangenome_clusters_First_genomes, '_')
|
|
129
|
-
else:
|
|
130
|
-
pangenome_clusters_Type = single_clustering_counting(pangenome_clusters_First, reps)
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
Number_Of_Second_Extending_But_Same_Genomes = 0
|
|
135
|
-
|
|
136
|
-
sorted_first_keys = sort_keys_by_values(pangenome_clusters_First, pangenome_clusters_First_sequences)
|
|
137
|
-
pangenome_clusters_First_sorted = reorder_dict_by_keys(pangenome_clusters_First, sorted_first_keys)
|
|
138
|
-
pangenome_clusters_First_sequences_sorted = reorder_dict_by_keys(pangenome_clusters_First_sequences, sorted_first_keys)
|
|
139
|
-
pangenome_clusters_Type_sorted = reorder_dict_by_keys(pangenome_clusters_Type, sorted_first_keys)
|
|
140
|
-
|
|
141
|
-
print("Calculating Groups")
|
|
142
|
-
seen_groupings = []
|
|
143
|
-
for cluster, numbers in pangenome_clusters_Type_sorted.items():
|
|
144
|
-
############################### Calculate First only
|
|
145
|
-
cluster = str(cluster)
|
|
146
|
-
for grouping in numbers[2]: #!!# Could do with a more elegant solution
|
|
147
|
-
current_cluster = grouping[0].split(':')[0]
|
|
148
|
-
if current_cluster not in seen_groupings:
|
|
149
|
-
seen_groupings.append(current_cluster)
|
|
150
|
-
current_cluster_size = grouping[0].split(':')[1]
|
|
151
|
-
calc_First_only_core(current_cluster, current_cluster_size, cores)
|
|
152
|
-
############################# Calculate First and Reclustered-Second
|
|
153
|
-
if numbers[0] == 1 and numbers[3] >= 1: # If Seconds did not combine First reps
|
|
154
|
-
calc_single_First_extended_Second_only_core(cluster, numbers[1], cores, numbers[3])
|
|
155
|
-
elif numbers[0] > 1 and numbers[3] >= 1: # If unique Seconds combined multiple Firsts
|
|
156
|
-
calc_multi_First_extended_Second_only_core(cluster, numbers[1], cores, numbers[3])
|
|
157
|
-
elif numbers[4] >= 1:
|
|
158
|
-
Number_Of_Second_Extending_But_Same_Genomes += 1
|
|
159
|
-
else:
|
|
160
|
-
if options.verbose == True:
|
|
161
|
-
print("First cluster " + current_cluster + " already processed - This is likely because it was clustered with another First representative.")
|
|
162
|
-
|
|
163
|
-
if options.reclustered != None:
|
|
164
|
-
combined_pangenome_clusters_ONLY_Second_Type = defaultdict(list)
|
|
165
|
-
combined_pangenome_clusters_Second_Type = defaultdict(list)
|
|
166
|
-
for cluster, genomes in combined_pangenome_clusters_Second.items():
|
|
167
|
-
if cluster in not_Second_only_cluster_ids:
|
|
168
|
-
combined_pangenome_clusters_Second_Type[cluster] = [cluster, len(genomes)]
|
|
169
|
-
else:
|
|
170
|
-
combined_pangenome_clusters_ONLY_Second_Type[cluster] = [cluster, len(genomes)]
|
|
171
|
-
for cluster, data in combined_pangenome_clusters_Second_Type.items():
|
|
172
|
-
if data[1] >= 1:
|
|
173
|
-
calc_Second_only_core(cluster, cores, data[1])
|
|
174
|
-
for cluster, data in combined_pangenome_clusters_ONLY_Second_Type.items():
|
|
175
|
-
if data[1] >= 1:
|
|
176
|
-
calc_only_Second_only_core(cluster, cores, data[1])
|
|
177
|
-
###########################
|
|
178
|
-
### Output
|
|
179
|
-
output_path = os.path.abspath(options.output_dir)
|
|
180
|
-
if not os.path.exists(output_path):
|
|
181
|
-
os.makedirs(output_path)
|
|
182
|
-
stats_out = os.path.join(output_path,'summary_statistics.txt')
|
|
183
|
-
key_order = list(cores.keys())
|
|
184
|
-
with open(stats_out,'w') as outfile:
|
|
185
|
-
print("Genus Groups:")
|
|
186
|
-
outfile.write("Genus Groups:\n")
|
|
187
|
-
for key in key_order:
|
|
188
|
-
print(key+':\t'+str(len(cores[key])))
|
|
189
|
-
outfile.write(key + ':\t' + str(len(cores[key]))+'\n')
|
|
190
|
-
print("Total Number of First Gene Groups (Including Singletons): " + str(len(pangenome_clusters_First_sequences_sorted)))
|
|
191
|
-
outfile.write("Total Number of Gene Groups (Including Singletons): " + str(len(pangenome_clusters_First_sequences_sorted)))
|
|
192
|
-
if options.reclustered!= None:
|
|
193
|
-
print("Total Number of Second Gene Groups (Including Singletons): " + str(
|
|
194
|
-
len(combined_pangenome_clusters_Second_sequences)))
|
|
195
|
-
print("Total Number of First Gene Groups That Had Additional Second Sequences But Not New Genomes: " + str(
|
|
196
|
-
Number_Of_Second_Extending_But_Same_Genomes))
|
|
197
|
-
outfile.write("\nTotal Number of Second Gene Groups (Including Singletons): " + str(
|
|
198
|
-
len(combined_pangenome_clusters_Second_sequences)))
|
|
199
|
-
outfile.write("\nTotal Number of First Gene Groups That Had Additional Second Sequences But Not New Genomes: " + str(
|
|
200
|
-
Number_Of_Second_Extending_But_Same_Genomes))
|
|
201
|
-
|
|
202
|
-
if options.gene_presence_absence_out != False:
|
|
203
|
-
gene_presence_absence_output(options,genus_dict, pangenome_clusters_First_sorted, pangenome_clusters_First_sequences_sorted)
|
|
204
|
-
|
|
205
|
-
if options.run_mode == 'Full':
|
|
206
|
-
if options.reclustered == None:
|
|
207
|
-
combined_pangenome_clusters_Second_sequences = None
|
|
208
|
-
if options.write_groups != None:
|
|
209
|
-
print("Outputting gene group FASTA files")
|
|
210
|
-
sequences = read_fasta(options.fasta)
|
|
211
|
-
#output_dir = os.path.dirname(os.path.abspath(options.output_dir))
|
|
212
|
-
output_dir = os.path.join(options.output_dir, 'Gene_Families_Output')
|
|
213
|
-
write_groups(options,output_dir, key_order, cores, sequences,
|
|
214
|
-
pangenome_clusters_First_sequences_sorted, combined_pangenome_clusters_Second_sequences)
|
|
215
|
-
|
|
216
|
-
elif options.run_mode == 'Partial':
|
|
217
|
-
if options.reclustered == None:
|
|
218
|
-
combined_pangenome_clusters_Second_sequences = None
|
|
219
|
-
if options.write_groups != None and options.fasta != None:
|
|
220
|
-
print("Outputting gene group FASTA files")
|
|
221
|
-
sequences = read_fasta(options.fasta)
|
|
222
|
-
#output_dir = os.path.dirname(os.path.abspath(options.output_dir))
|
|
223
|
-
output_dir = os.path.join(options.output_dir, 'Gene_Families_Output')
|
|
224
|
-
write_groups(options,output_dir, key_order, cores, sequences,
|
|
225
|
-
pangenome_clusters_First_sequences_sorted, combined_pangenome_clusters_Second_sequences)
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
# if options.write_groups != None and options.fasta != None:
|
|
229
|
-
# sequences = read_fasta(options.fasta)
|
|
230
|
-
# output_dir = os.path.join(output_path, 'Gene_Families_Output')
|
|
231
|
-
#
|
|
232
|
-
# write_groups(options,output_dir, key_order, cores, sequences,
|
|
233
|
-
# pangenome_clusters_First_sequences_sorted, combined_pangenome_clusters_Second_sequences)
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
#!!# - Currently only align in Species Mode
|
|
237
|
-
#if options.align_core != None and options.fasta != None and options.write_groups != None:
|
|
238
|
-
# process_gene_families(options, os.path.join(output_path, 'Gene_Families_Output'), 'concatenated_genes_aligned.fasta')
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|