drep 4.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- drep/VERSION +1 -0
- drep/WorkDirectory.py +355 -0
- drep/__init__.py +101 -0
- drep/argumentParser.py +279 -0
- drep/controller.py +105 -0
- drep/d_adjust.py +272 -0
- drep/d_analyze.py +1613 -0
- drep/d_bonus.py +429 -0
- drep/d_choose.py +362 -0
- drep/d_cluster/__init__.py +0 -0
- drep/d_cluster/cluster_utils.py +126 -0
- drep/d_cluster/compare_utils.py +636 -0
- drep/d_cluster/controller.py +228 -0
- drep/d_cluster/external.py +765 -0
- drep/d_cluster/greedy_clustering.py +181 -0
- drep/d_cluster/parsers.py +0 -0
- drep/d_cluster/union_find.py +543 -0
- drep/d_cluster/utils.py +687 -0
- drep/d_evaluate.py +355 -0
- drep/d_filter.py +831 -0
- drep/d_workflows.py +135 -0
- drep-4.0.2.data/scripts/ScaffoldLevel_dRep.py +1101 -0
- drep-4.0.2.data/scripts/dRep +32 -0
- drep-4.0.2.data/scripts/parse_stb.py +140 -0
- drep-4.0.2.dist-info/METADATA +23 -0
- drep-4.0.2.dist-info/RECORD +28 -0
- drep-4.0.2.dist-info/WHEEL +5 -0
- drep-4.0.2.dist-info/top_level.txt +1 -0
drep/d_cluster/utils.py
ADDED
|
@@ -0,0 +1,687 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
'''
|
|
3
|
+
d_cluster - a subset of dRep
|
|
4
|
+
|
|
5
|
+
Clusters a list of genomes with both primary and secondary clustering
|
|
6
|
+
'''
|
|
7
|
+
|
|
8
|
+
import pandas as pd
|
|
9
|
+
import os
|
|
10
|
+
import logging
|
|
11
|
+
import random
|
|
12
|
+
import string
|
|
13
|
+
import numpy as np
|
|
14
|
+
|
|
15
|
+
import drep
|
|
16
|
+
import drep.d_cluster.cluster_utils
|
|
17
|
+
import drep.d_cluster.compare_utils
|
|
18
|
+
import drep.d_filter
|
|
19
|
+
import drep.d_bonus
|
|
20
|
+
|
|
21
|
+
# from drep.d_cluster import genomeChunk
|
|
22
|
+
import drep.d_cluster.controller
|
|
23
|
+
#from drep.d_cluster.controller import genomeChunk, genome_hierarchical_clustering, iteratre_clusters, cluster_hierarchical
|
|
24
|
+
|
|
25
|
+
# This is to make pandas shut up with it's warnings
|
|
26
|
+
pd.options.mode.chained_assignment = None
|
|
27
|
+
|
|
28
|
+
import drep.d_cluster.external
|
|
29
|
+
#from drep.d_cluster.external import parse_gani_file, parse_nsim_file, gen_nucmer_cmd, add_avani
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _cluster_Ndb(Ndb, id='primary_cluster', **kwargs):
|
|
34
|
+
'''
|
|
35
|
+
Cluster Ndb on id
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
Ndb: obvious
|
|
39
|
+
id: thing that defines subclusters in Ndb (defauls = 'primary_cluster')
|
|
40
|
+
|
|
41
|
+
Keyword arguments:
|
|
42
|
+
all: passed on to genome_hierarchical_clustering
|
|
43
|
+
|
|
44
|
+
Returns:
|
|
45
|
+
list: [Cdb, c2ret(cluster name -> clusering files)]
|
|
46
|
+
'''
|
|
47
|
+
Cdb = pd.DataFrame()
|
|
48
|
+
c2ret = {}
|
|
49
|
+
for name, ndb in Ndb.groupby(id):
|
|
50
|
+
cdb, cluster_ret = drep.d_cluster.cluster_utils.genome_hierarchical_clustering(ndb, cluster=name, **kwargs)
|
|
51
|
+
cdb[id] = name
|
|
52
|
+
Cdb = pd.concat([Cdb,cdb], ignore_index=True)
|
|
53
|
+
c2ret[name] = cluster_ret
|
|
54
|
+
|
|
55
|
+
return Cdb, c2ret
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def make_linkage_Ndb(Ndb, **kwargs):
|
|
59
|
+
'''
|
|
60
|
+
Filter the Ndb in accordance with the kwargs. Average reciprical ANI values
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
Ndb: result of secondary clutsering
|
|
64
|
+
|
|
65
|
+
Keyword arguments:
|
|
66
|
+
cov_thresh: minimum coverage threshold (default = 0.5)
|
|
67
|
+
|
|
68
|
+
Return:
|
|
69
|
+
DataFrame: pivoted DataFrame ready for clustering
|
|
70
|
+
'''
|
|
71
|
+
d = Ndb.copy()
|
|
72
|
+
cov_thresh = float(kwargs.get('cov_thresh',0.5))
|
|
73
|
+
|
|
74
|
+
# Remove values without enough coverage
|
|
75
|
+
d.loc[d['alignment_coverage'] <= cov_thresh, 'ani'] = 0
|
|
76
|
+
|
|
77
|
+
# Make a linkagedb by averaging values and setting self-compare to 1
|
|
78
|
+
drep.d_cluster.external.add_avani(d)
|
|
79
|
+
d['dist'] = 1 - d['av_ani']
|
|
80
|
+
db = d.pivot(index="reference", columns="querry", values="dist")
|
|
81
|
+
|
|
82
|
+
return db
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def estimate_time(comps, alg):
|
|
86
|
+
'''
|
|
87
|
+
Estimate time, in minutes, based on comparison algorithm and number of comparisons
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
comps: number of genomes comparisons to perform
|
|
91
|
+
alg: algorthm used
|
|
92
|
+
|
|
93
|
+
Return:
|
|
94
|
+
float: time to perfom comparison (in minutes)
|
|
95
|
+
'''
|
|
96
|
+
# Minutes per comparison, very roughly. This only drives a log message, so an
|
|
97
|
+
# unknown algorithm must never take the run down with it -- fall back to the
|
|
98
|
+
# fastest estimate rather than raising.
|
|
99
|
+
per_comparison = {
|
|
100
|
+
'ANIn': .33,
|
|
101
|
+
'gANI': .1,
|
|
102
|
+
'goANI': .1,
|
|
103
|
+
'ANImf': .5,
|
|
104
|
+
'fastANI': 0.00667,
|
|
105
|
+
'skani': 0.00667,
|
|
106
|
+
}
|
|
107
|
+
if alg not in per_comparison:
|
|
108
|
+
logging.debug(f"No time estimate available for {alg}; assuming a fast algorithm")
|
|
109
|
+
return comps * per_comparison.get(alg, 0.00667)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _gen_cdb_from_fclust(fclust,names):
|
|
115
|
+
'''
|
|
116
|
+
Make Cdb from the result of scipy.cluster.hierarchy.fcluster
|
|
117
|
+
|
|
118
|
+
Args:
|
|
119
|
+
fclust: result of scipy.cluster.hierarchy.fcluster
|
|
120
|
+
names: list(db.columns) of the input dataframe
|
|
121
|
+
|
|
122
|
+
Returns:
|
|
123
|
+
DataFrame: Cdb
|
|
124
|
+
'''
|
|
125
|
+
Table={'cluster':[],'genome':[]}
|
|
126
|
+
for i, c in enumerate(fclust):
|
|
127
|
+
Table['cluster'].append(c)
|
|
128
|
+
Table['genome'].append(names[i])
|
|
129
|
+
|
|
130
|
+
return pd.DataFrame(Table)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def parse_mash_table(location):
|
|
134
|
+
try:
|
|
135
|
+
iniCols = ['genome1', 'genome2', 'dist', 'p', 'kmers']
|
|
136
|
+
uCols = ['genome1', 'genome2', 'dist']
|
|
137
|
+
dTypes = {'genome1': 'category', 'genome2': 'category', 'dist': np.float32}
|
|
138
|
+
Mdb = pd.read_csv(location, names=iniCols, usecols=uCols, dtype=dTypes, sep='\t')
|
|
139
|
+
except ValueError:
|
|
140
|
+
logging.error("You're using an incompatible version of Mash! I'll try and fix it, but I recommend upgrading Mash to v2.2 or downgrading to v1 if problems occur")
|
|
141
|
+
Mdb = pd.read_csv(location, sep='\t')
|
|
142
|
+
|
|
143
|
+
Mdb['genome1'] = Mdb['genome1'].apply(_get_genome_name_from_fasta).astype('category')
|
|
144
|
+
Mdb['genome2'] = Mdb['genome2'].apply(_get_genome_name_from_fasta).astype('category')
|
|
145
|
+
Mdb['similarity'] = 1 - Mdb['dist']
|
|
146
|
+
|
|
147
|
+
return Mdb
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def merge_genome_chunks(mash_exe, genome_chunks, sketch_folder, MASH_folder):
|
|
151
|
+
all_file = os.path.join(MASH_folder, 'ALL.msh')
|
|
152
|
+
cmd = [mash_exe, 'paste', all_file] + [gc.all_file for gc in genome_chunks]
|
|
153
|
+
|
|
154
|
+
new_locs = []
|
|
155
|
+
new_names = []
|
|
156
|
+
for gc in genome_chunks:
|
|
157
|
+
new_locs += gc.genome_locations
|
|
158
|
+
new_names += gc.genome_names
|
|
159
|
+
|
|
160
|
+
new_gc = drep.d_cluster.compare_utils.genomeChunk(new_locs, 'all', sketch_folder, new_names, no_create=True)
|
|
161
|
+
new_gc.all_file = all_file
|
|
162
|
+
|
|
163
|
+
return cmd, new_gc
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
# def all_vs_all_MASH(Bdb, data_folder, **kwargs):
|
|
167
|
+
# """
|
|
168
|
+
# Run MASH pairwise within all samples in Bdb
|
|
169
|
+
#
|
|
170
|
+
# Args:
|
|
171
|
+
# Bdb: dataframe with genome, location
|
|
172
|
+
# data_folder: location to store output files
|
|
173
|
+
#
|
|
174
|
+
# Keyword Args:
|
|
175
|
+
# MASH_sketch: size of mash sketches
|
|
176
|
+
# dry: dont actually run anything
|
|
177
|
+
# processors: number of processors to multithread with
|
|
178
|
+
# mash_exe: location of mash excutible (will try and find with shutil if not provided)
|
|
179
|
+
# groupSize: max number of mash sketches to hold in each folder
|
|
180
|
+
# debug: if True, log all of the commands
|
|
181
|
+
# wd: if you want to log commands, you also need the wd
|
|
182
|
+
# """
|
|
183
|
+
#
|
|
184
|
+
# MASH_s = kwargs.get('MASH_sketch',1000)
|
|
185
|
+
# dry = kwargs.get('dry',False)
|
|
186
|
+
# # overwrite = kwargs.get('overwrite', False)
|
|
187
|
+
# p = kwargs.get('processors',6)
|
|
188
|
+
# groupSize = kwargs.get('groupSize', 1000)
|
|
189
|
+
#
|
|
190
|
+
# # set up logdir
|
|
191
|
+
# if ('wd' in kwargs) and (kwargs.get('debug', False) == True):
|
|
192
|
+
# logdir = kwargs.get('wd').get_dir('cmd_logs')
|
|
193
|
+
# else:
|
|
194
|
+
# logdir = False
|
|
195
|
+
#
|
|
196
|
+
# # Find mash
|
|
197
|
+
# mash_exe = kwargs.get('exe_loc', None)
|
|
198
|
+
# if mash_exe == None:
|
|
199
|
+
# mash_exe = drep.get_exe('mash')
|
|
200
|
+
#
|
|
201
|
+
# # Set up folders
|
|
202
|
+
# MASH_folder = os.path.join(data_folder, 'MASH_files/')
|
|
203
|
+
# if not os.path.exists(MASH_folder):
|
|
204
|
+
# os.makedirs(MASH_folder)
|
|
205
|
+
#
|
|
206
|
+
# sketch_folder = os.path.join(MASH_folder, 'sketches/')
|
|
207
|
+
# if not os.path.exists(sketch_folder):
|
|
208
|
+
# os.makedirs(sketch_folder)
|
|
209
|
+
#
|
|
210
|
+
# # Make chunks
|
|
211
|
+
# l2g = Bdb.set_index('location')['genome'].to_dict()
|
|
212
|
+
# locations = list(Bdb['location'].unique())
|
|
213
|
+
# chunks = [locations[x:x+groupSize] for x in range(0, len(locations), groupSize)]
|
|
214
|
+
#
|
|
215
|
+
# # Make the MASH sketches
|
|
216
|
+
# cmds = []
|
|
217
|
+
# chunk_folders = []
|
|
218
|
+
# for i, chunk in enumerate(chunks):
|
|
219
|
+
# chunk_folder = os.path.join(sketch_folder, "chunk_{0}".format(i))
|
|
220
|
+
# chunk_folders.append(chunk_folder)
|
|
221
|
+
# if not os.path.exists(chunk_folder):
|
|
222
|
+
# os.makedirs(chunk_folder)
|
|
223
|
+
# for fasta in chunk:
|
|
224
|
+
# genome = l2g[fasta]
|
|
225
|
+
# file = os.path.join(chunk_folder, genome)
|
|
226
|
+
# if not os.path.isfile(file + '.msh'):
|
|
227
|
+
# cmd = [mash_exe, 'sketch', fasta, '-s', str(MASH_s), '-o',
|
|
228
|
+
# file]
|
|
229
|
+
# cmds.append(cmd)
|
|
230
|
+
#
|
|
231
|
+
# if not dry:
|
|
232
|
+
# if len(cmds) > 0:
|
|
233
|
+
# drep.thread_cmds(cmds, logdir=logdir, t=int(p))
|
|
234
|
+
#
|
|
235
|
+
# # Combine MASH sketches within chunk
|
|
236
|
+
# cmds = []
|
|
237
|
+
# alls = []
|
|
238
|
+
# for chunk_folder in chunk_folders:
|
|
239
|
+
# all_file = os.path.join(chunk_folder, 'chunk_all.msh')
|
|
240
|
+
# cmd = [mash_exe, 'paste', all_file] \
|
|
241
|
+
# + glob.glob(os.path.join(chunk_folder, '*'))
|
|
242
|
+
# cmds.append(cmd)
|
|
243
|
+
# alls.append(all_file)
|
|
244
|
+
# if not dry:
|
|
245
|
+
# if len(cmds) > 0:
|
|
246
|
+
# drep.thread_cmds(cmds, logdir=logdir, t=int(p))
|
|
247
|
+
#
|
|
248
|
+
# # Combine MASH sketches of all chunks
|
|
249
|
+
# all_file = os.path.join(MASH_folder, 'ALL.msh')
|
|
250
|
+
# cmd = [mash_exe, 'paste', all_file] + alls
|
|
251
|
+
# drep.run_cmd(cmd, dry, shell=False, logdir=logdir)
|
|
252
|
+
#
|
|
253
|
+
# # Calculate distances
|
|
254
|
+
# cmd = [mash_exe, 'dist','-p', str(p), all_file, all_file, '>', MASH_folder
|
|
255
|
+
# + 'MASH_table.tsv']
|
|
256
|
+
# cmd = ' '.join(cmd)
|
|
257
|
+
# drep.run_cmd(cmd, dry, shell=True, logdir=logdir)
|
|
258
|
+
#
|
|
259
|
+
# # Make Mdb based on all genomes in the MASH folder
|
|
260
|
+
# file = MASH_folder + 'MASH_table.tsv'
|
|
261
|
+
#
|
|
262
|
+
# iniCols = ['genome1','genome2','dist','p','kmers']
|
|
263
|
+
# uCols = ['genome1','genome2','dist']
|
|
264
|
+
# dTypes = {'genome1':'category', 'genome2':'category', 'dist':np.float32}
|
|
265
|
+
# Mdb = pd.read_csv(file, names=iniCols, usecols=uCols, dtype=dTypes, sep='\t')
|
|
266
|
+
# Mdb['genome1'] = Mdb['genome1'].apply(_get_genome_name_from_fasta).astype('category')
|
|
267
|
+
# Mdb['genome2'] = Mdb['genome2'].apply(_get_genome_name_from_fasta).astype('category')
|
|
268
|
+
# Mdb['similarity'] = 1 - Mdb['dist']
|
|
269
|
+
#
|
|
270
|
+
# # Filter out those genomes that are in the MASH folder but shouldn't be in Mdb
|
|
271
|
+
# genomes = Bdb['genome'].unique()
|
|
272
|
+
# Mdb = Mdb[Mdb['genome1'].isin(genomes)]
|
|
273
|
+
# Mdb = Mdb[Mdb['genome2'].isin(genomes)]
|
|
274
|
+
#
|
|
275
|
+
# # Reorder categories to be correct
|
|
276
|
+
# for g in ['genome1', 'genome2']:
|
|
277
|
+
# Mdb[g] = Mdb[g].cat.remove_unused_categories()
|
|
278
|
+
# Mdb[g] = Mdb[g].cat.reorder_categories(sorted((Mdb[g].unique())), ordered=True)
|
|
279
|
+
#
|
|
280
|
+
# return Mdb
|
|
281
|
+
|
|
282
|
+
def _get_genome_name_from_fasta(fasta):
|
|
283
|
+
'''
|
|
284
|
+
Just do a os.path.basename command
|
|
285
|
+
|
|
286
|
+
Args:
|
|
287
|
+
fasta: location of .fasta file
|
|
288
|
+
|
|
289
|
+
Returns:
|
|
290
|
+
string: basename of .fasta files
|
|
291
|
+
'''
|
|
292
|
+
return str(os.path.basename(fasta))
|
|
293
|
+
|
|
294
|
+
def parse_delta(filename):
|
|
295
|
+
'''
|
|
296
|
+
Parse a .delta file from nucmer
|
|
297
|
+
|
|
298
|
+
Args:
|
|
299
|
+
filename: location of .delta file
|
|
300
|
+
|
|
301
|
+
Returns:
|
|
302
|
+
list: [alignment_length, similarity_errors]
|
|
303
|
+
'''
|
|
304
|
+
|
|
305
|
+
aln_length, sim_errors = 0, 0
|
|
306
|
+
for line in [l.strip().split() for l in open(filename, 'r').readlines()]:
|
|
307
|
+
if line[0] == 'NUCMER' or line[0].startswith('>'): # Skip headers
|
|
308
|
+
continue
|
|
309
|
+
# We only process lines with seven columns:
|
|
310
|
+
if len(line) == 7:
|
|
311
|
+
aln_length += abs(int(line[1]) - int(line[0]))
|
|
312
|
+
sim_errors += int(line[4])
|
|
313
|
+
return aln_length, sim_errors
|
|
314
|
+
|
|
315
|
+
def gen_gANI_cmd(file, g1, g2, exe):
|
|
316
|
+
'''
|
|
317
|
+
Generate a command to run gANI (ANIcalculator)
|
|
318
|
+
|
|
319
|
+
Will return [] if g1 == g2
|
|
320
|
+
|
|
321
|
+
Args:
|
|
322
|
+
file: output file name
|
|
323
|
+
g1: location of the genes of genome1
|
|
324
|
+
g2: location of the genes of genome2
|
|
325
|
+
exe: location of gANI executible
|
|
326
|
+
|
|
327
|
+
Returns:
|
|
328
|
+
list: command to run
|
|
329
|
+
'''
|
|
330
|
+
# Don't do self-comparions
|
|
331
|
+
if g1 == g2:
|
|
332
|
+
return []
|
|
333
|
+
|
|
334
|
+
else:
|
|
335
|
+
cmd = [exe,'-genome1fna',g1,'-genome2fna',g2,'-outfile',file,'-outdir',file + 'TEMP']
|
|
336
|
+
return cmd
|
|
337
|
+
|
|
338
|
+
def gen_goANI_cmd(file, g1, g2, exe):
|
|
339
|
+
'''
|
|
340
|
+
Generate a command to run goANI
|
|
341
|
+
|
|
342
|
+
Will return [] if g1 == g2
|
|
343
|
+
|
|
344
|
+
Args:
|
|
345
|
+
file: output file name
|
|
346
|
+
g1: location of the genes of genome1
|
|
347
|
+
g2: location of the genes of genome2
|
|
348
|
+
exe: location of nsimscan executible
|
|
349
|
+
|
|
350
|
+
Returns:
|
|
351
|
+
list: command to run
|
|
352
|
+
'''
|
|
353
|
+
# Don't do self-comparions
|
|
354
|
+
if g1 == g2:
|
|
355
|
+
return []
|
|
356
|
+
|
|
357
|
+
else:
|
|
358
|
+
cmd = [exe,'--om','TABX', '--rpq', '1', g1, g2, file]
|
|
359
|
+
return cmd
|
|
360
|
+
|
|
361
|
+
def process_deltafiles(deltafiles, org_lengths, logger=None, **kwargs):
|
|
362
|
+
'''
|
|
363
|
+
Parse a list of delta files into a pandas dataframe
|
|
364
|
+
|
|
365
|
+
Files NEED to be named in the format genome1_vs_genome2.delta, or genome1_vs_genome2.filtered.delta
|
|
366
|
+
|
|
367
|
+
Args:
|
|
368
|
+
deltafiles: list of .delta files
|
|
369
|
+
org_lengths: dictionary of genome to genome length
|
|
370
|
+
|
|
371
|
+
Keyword Arguments:
|
|
372
|
+
coverage_method: default is larger
|
|
373
|
+
logger: if not None, will log 0 errors
|
|
374
|
+
|
|
375
|
+
Returns:
|
|
376
|
+
DataFrame: information about the alignments
|
|
377
|
+
'''
|
|
378
|
+
|
|
379
|
+
Table = {'querry':[],'reference':[],'alignment_length':[],'similarity_errors':[],
|
|
380
|
+
'ref_coverage':[],'querry_coverage':[],'ani':[], 'reference_length':[],
|
|
381
|
+
'querry_length':[],'alignment_coverage':[]}
|
|
382
|
+
|
|
383
|
+
coverage_method = kwargs.get('coverage_method', 'larger')
|
|
384
|
+
|
|
385
|
+
for deltafile in deltafiles:
|
|
386
|
+
qname, sname = os.path.splitext(os.path.split(deltafile)[-1])[0].split('_vs_')
|
|
387
|
+
# Handle naming of "filtered"
|
|
388
|
+
if sname.endswith('.delta'):
|
|
389
|
+
sname = sname[:-6]
|
|
390
|
+
|
|
391
|
+
tot_length, tot_sim_error = parse_delta(deltafile)
|
|
392
|
+
if tot_length == 0 and logger is not None:
|
|
393
|
+
logging.info("Total alignment length reported in " +
|
|
394
|
+
"%s is zero!" % deltafile)
|
|
395
|
+
query_cover = float(tot_length) / org_lengths[qname]
|
|
396
|
+
sbjct_cover = float(tot_length) / org_lengths[sname]
|
|
397
|
+
|
|
398
|
+
try:
|
|
399
|
+
perc_id = 1 - float(tot_sim_error) / tot_length
|
|
400
|
+
except ZeroDivisionError:
|
|
401
|
+
#print("Alignment between {0} and {1} has 0 alignment!".format(qname,sname))
|
|
402
|
+
perc_id = 0 # set arbitrary value of zero identity
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
Table['querry'].append(qname)
|
|
406
|
+
Table['querry_length'].append(org_lengths[qname])
|
|
407
|
+
Table['reference'].append(sname)
|
|
408
|
+
Table['reference_length'].append(org_lengths[sname])
|
|
409
|
+
Table['alignment_length'].append(tot_length)
|
|
410
|
+
Table['similarity_errors'].append(tot_sim_error)
|
|
411
|
+
Table['ani'].append(perc_id)
|
|
412
|
+
Table['ref_coverage'].append(sbjct_cover)
|
|
413
|
+
Table['querry_coverage'].append(query_cover)
|
|
414
|
+
|
|
415
|
+
if coverage_method == 'total':
|
|
416
|
+
Table['alignment_coverage'].append((tot_length * 2)/(org_lengths[qname]\
|
|
417
|
+
+ org_lengths[sname]))
|
|
418
|
+
elif coverage_method == 'larger':
|
|
419
|
+
Table['alignment_coverage'].append(max((tot_length/org_lengths[qname]),\
|
|
420
|
+
(tot_length/org_lengths[sname])))
|
|
421
|
+
|
|
422
|
+
dTypes = {'querry':'category','querry_length':np.int64, 'reference':'category',\
|
|
423
|
+
'reference_length':np.int64, 'alignment_length':np.int64, 'similarity_errors':np.int64,\
|
|
424
|
+
'ani':np.float32, 'ref_coverage':np.float32, 'querry_coverage':np.float32}
|
|
425
|
+
df = pd.DataFrame(Table)
|
|
426
|
+
for c, t in dTypes.items():
|
|
427
|
+
df[c] = df[c].astype(t)
|
|
428
|
+
return df
|
|
429
|
+
|
|
430
|
+
def process_goani_files(files):
|
|
431
|
+
'''
|
|
432
|
+
From a list of nsimscan files, return a parsed DataFrame like normal,
|
|
433
|
+
and a special one for calculating dn/ds
|
|
434
|
+
|
|
435
|
+
Args:
|
|
436
|
+
list: files
|
|
437
|
+
|
|
438
|
+
Returns:
|
|
439
|
+
DataFrame: Ndb
|
|
440
|
+
'''
|
|
441
|
+
Table = {'querry':[],'reference':[],'ani':[],'alignment_coverage':[]}
|
|
442
|
+
for file in files:
|
|
443
|
+
results = drep.d_cluster.external.parse_nsim_file(file)
|
|
444
|
+
results['reference'] = os.path.basename(file).split('_vs_')[0]
|
|
445
|
+
results['querry'] = os.path.basename(file).split('_vs_')[1][:-4]
|
|
446
|
+
|
|
447
|
+
# Add forward results
|
|
448
|
+
Table['reference'].append(results['reference'])
|
|
449
|
+
Table['querry'].append(results['querry'])
|
|
450
|
+
Table['ani'].append(float(results['ani'])/100)
|
|
451
|
+
Table['alignment_coverage'].append(results['af'])
|
|
452
|
+
#Table['alignment_coverage'].append(results['rq_coverage'])
|
|
453
|
+
|
|
454
|
+
# Add reverse results [they're the same]
|
|
455
|
+
# Table['reference'].append(results['querry'])
|
|
456
|
+
# Table['querry'].append(results['reference'])
|
|
457
|
+
# Table['ani'].append(float(results['ani'])/100)
|
|
458
|
+
# Table['alignment_coverage'].append(results['af'])
|
|
459
|
+
#Table['alignment_coverage'].append(results['qr_coverage'])
|
|
460
|
+
|
|
461
|
+
# Add self comparisons
|
|
462
|
+
for g in set(Table['reference']):
|
|
463
|
+
Table['reference'].append(g)
|
|
464
|
+
Table['querry'].append(g)
|
|
465
|
+
Table['ani'].append(1)
|
|
466
|
+
Table['alignment_coverage'].append(1)
|
|
467
|
+
|
|
468
|
+
Gdb = pd.DataFrame(Table)
|
|
469
|
+
return Gdb
|
|
470
|
+
|
|
471
|
+
def process_gani_files(files):
|
|
472
|
+
'''
|
|
473
|
+
From a list of gANI output files, return a parsed DataFrame
|
|
474
|
+
|
|
475
|
+
Args:
|
|
476
|
+
list: files
|
|
477
|
+
|
|
478
|
+
Returns:
|
|
479
|
+
DataFrame: Ndb
|
|
480
|
+
'''
|
|
481
|
+
Table = {'querry':[],'reference':[],'ani':[],'alignment_coverage':[]}
|
|
482
|
+
for file in files:
|
|
483
|
+
results = drep.d_cluster.external.parse_gani_file(file)
|
|
484
|
+
|
|
485
|
+
# calculate coverage using the "larger" method
|
|
486
|
+
cov = max(results['rq_coverage'], results['qr_coverage'])
|
|
487
|
+
|
|
488
|
+
# Add forward results
|
|
489
|
+
Table['reference'].append(results['reference'])
|
|
490
|
+
Table['querry'].append(results['querry'])
|
|
491
|
+
Table['ani'].append(float(results['rq_ani'])/100)
|
|
492
|
+
Table['alignment_coverage'].append(cov)
|
|
493
|
+
#Table['alignment_coverage'].append(results['rq_coverage'])
|
|
494
|
+
|
|
495
|
+
# Add reverse results [they're the same]
|
|
496
|
+
Table['reference'].append(results['querry'])
|
|
497
|
+
Table['querry'].append(results['reference'])
|
|
498
|
+
Table['ani'].append(float(results['qr_ani'])/100)
|
|
499
|
+
Table['alignment_coverage'].append(cov)
|
|
500
|
+
#Table['alignment_coverage'].append(results['qr_coverage'])
|
|
501
|
+
|
|
502
|
+
# Add self comparisons
|
|
503
|
+
for g in set(Table['reference']):
|
|
504
|
+
Table['reference'].append(g)
|
|
505
|
+
Table['querry'].append(g)
|
|
506
|
+
Table['ani'].append(1)
|
|
507
|
+
Table['alignment_coverage'].append(1)
|
|
508
|
+
|
|
509
|
+
Gdb = pd.DataFrame(Table)
|
|
510
|
+
return Gdb
|
|
511
|
+
|
|
512
|
+
def run_pairwise_ANIn(genome_list, ANIn_folder, **kwargs):
|
|
513
|
+
'''
|
|
514
|
+
Given a list of genomes and an output folder, compare all genomes using ANImf
|
|
515
|
+
|
|
516
|
+
Args:
|
|
517
|
+
genome_list: list of locations of genome files
|
|
518
|
+
ANIn_folder: folder to store the output of comparison
|
|
519
|
+
|
|
520
|
+
Keyword arguments:
|
|
521
|
+
processors: threads to use
|
|
522
|
+
debug: if true save extra output
|
|
523
|
+
wd: needed if debug is True
|
|
524
|
+
'''
|
|
525
|
+
p = kwargs.get('processors',6)
|
|
526
|
+
genomes = genome_list
|
|
527
|
+
|
|
528
|
+
# Make folder
|
|
529
|
+
if not os.path.exists(ANIn_folder):
|
|
530
|
+
os.makedirs(ANIn_folder)
|
|
531
|
+
|
|
532
|
+
# Gen commands
|
|
533
|
+
cmds = []
|
|
534
|
+
files = []
|
|
535
|
+
for g1 in genomes:
|
|
536
|
+
# Make it so each reference is it's own folder, to spread out .delta files
|
|
537
|
+
cur_folder = os.path.join(ANIn_folder, _get_genome_name_from_fasta(g1))
|
|
538
|
+
if not os.path.exists(cur_folder):
|
|
539
|
+
os.makedirs(cur_folder)
|
|
540
|
+
|
|
541
|
+
for g2 in genomes:
|
|
542
|
+
file_name = "{0}{1}_vs_{2}".format(ANIn_folder, \
|
|
543
|
+
_get_genome_name_from_fasta(g1),\
|
|
544
|
+
_get_genome_name_from_fasta(g2))
|
|
545
|
+
files.append(file_name)
|
|
546
|
+
|
|
547
|
+
# If the file doesn't already exist, add it to what needs to be run
|
|
548
|
+
if not os.path.isfile(file_name + '.delta'):
|
|
549
|
+
cmds.append(drep.d_cluster.external.gen_nucmer_cmd(file_name, g1, g2))
|
|
550
|
+
|
|
551
|
+
# Run commands
|
|
552
|
+
if len(cmds) > 0:
|
|
553
|
+
for c in cmds:
|
|
554
|
+
logging.debug(' '.join(c))
|
|
555
|
+
|
|
556
|
+
if ('wd' in kwargs) and (kwargs.get('debug', False)):
|
|
557
|
+
logdir = kwargs.get('wd').get_dir('cmd_logs')
|
|
558
|
+
else:
|
|
559
|
+
logdir = False
|
|
560
|
+
drep.thread_cmds(cmds, shell=False, logdir=logdir, t=int(p))
|
|
561
|
+
|
|
562
|
+
# Make dictionary of genome lengths
|
|
563
|
+
org_lengths = {}
|
|
564
|
+
for genome in genomes:
|
|
565
|
+
org_lengths[_get_genome_name_from_fasta(genome)] = \
|
|
566
|
+
drep.d_filter.calc_fasta_length(genome)
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
deltafiles = ["{0}.delta".format(file) for file in files]
|
|
571
|
+
df = process_deltafiles(deltafiles, org_lengths, **kwargs)
|
|
572
|
+
|
|
573
|
+
return df
|
|
574
|
+
|
|
575
|
+
def load_genomes(genome_list):
|
|
576
|
+
'''
|
|
577
|
+
Takes a list of genome locations, returns a pandas dataframe with the
|
|
578
|
+
locations and the basename of the genomes
|
|
579
|
+
|
|
580
|
+
Args:
|
|
581
|
+
genome_list: list of genome locations
|
|
582
|
+
|
|
583
|
+
Returns:
|
|
584
|
+
DataFrame: pandas dataframe with columns ['genome', 'location']
|
|
585
|
+
'''
|
|
586
|
+
assert type(genome_list) == type(list())
|
|
587
|
+
|
|
588
|
+
# Load from a text file
|
|
589
|
+
if len(genome_list) == 1:
|
|
590
|
+
logging.info('Loading genomes from a list')
|
|
591
|
+
try:
|
|
592
|
+
Table = {'genome':[],'location':[]}
|
|
593
|
+
with open(genome_list[0], 'r') as o:
|
|
594
|
+
for line in o.readlines():
|
|
595
|
+
genome = line.strip()
|
|
596
|
+
assert os.path.isfile(genome), "{0} is not a file".format(genome)
|
|
597
|
+
Table['genome'].append(os.path.basename(genome))
|
|
598
|
+
Table['location'].append(os.path.abspath(genome))
|
|
599
|
+
Bdb = pd.DataFrame(Table)
|
|
600
|
+
return Bdb
|
|
601
|
+
except:
|
|
602
|
+
logging.info('Nevermind! Ill try loading as a genome now')
|
|
603
|
+
|
|
604
|
+
Table = {'genome':[], 'location':[]}
|
|
605
|
+
for genome in genome_list:
|
|
606
|
+
assert os.path.isfile(genome), "{0} is not a file".format(genome)
|
|
607
|
+
Table['genome'].append(os.path.basename(genome))
|
|
608
|
+
Table['location'].append(os.path.abspath(genome))
|
|
609
|
+
Bdb = pd.DataFrame(Table)
|
|
610
|
+
return Bdb
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
def _gen_nomani_cdb(Cdb, **kwargs):
|
|
614
|
+
'''
|
|
615
|
+
Make a Cdb that looks like the Cdb from secondary clustering
|
|
616
|
+
|
|
617
|
+
Args:
|
|
618
|
+
Cdb: Result from Mash clustering
|
|
619
|
+
|
|
620
|
+
Returns:
|
|
621
|
+
DataFrame: Cdb
|
|
622
|
+
'''
|
|
623
|
+
c_method = kwargs.get('clusterAlg','single')
|
|
624
|
+
threshold = 1 - kwargs.get('P_ani',.9)
|
|
625
|
+
|
|
626
|
+
# Make Cdb look like Cdb
|
|
627
|
+
cdb = Cdb.copy()
|
|
628
|
+
cdb['secondary_cluster'] = ["{0}_0".format(i) for i in cdb['primary_cluster']]
|
|
629
|
+
cdb.rename(columns={'primary_cluster': 'primary_cluster'}, inplace=True)
|
|
630
|
+
cdb['threshold'] = threshold
|
|
631
|
+
cdb['cluster_method'] = c_method
|
|
632
|
+
cdb['comparison_algorithm'] = 'MASH'
|
|
633
|
+
|
|
634
|
+
return cdb
|
|
635
|
+
|
|
636
|
+
def _print_time_estimate(Bdb, Cdb, algorithm, cores):
|
|
637
|
+
'''
|
|
638
|
+
Print an estimate of how long genome comaprisons will take
|
|
639
|
+
|
|
640
|
+
Args:
|
|
641
|
+
Bdb: DataFrame with [genome, location]
|
|
642
|
+
Cdb: Clustering DataFrame
|
|
643
|
+
algorthm: algorithm to estimate time with
|
|
644
|
+
'''
|
|
645
|
+
if algorithm != 'fastANI':
|
|
646
|
+
comps = 0
|
|
647
|
+
for bdb, name in drep.d_cluster.cluster_utils.iteratre_clusters(Bdb, Cdb):
|
|
648
|
+
g = len(bdb['genome'].unique())
|
|
649
|
+
comps += (g * g)
|
|
650
|
+
|
|
651
|
+
time = estimate_time(comps, algorithm)
|
|
652
|
+
time = time / int(cores)
|
|
653
|
+
|
|
654
|
+
else:
|
|
655
|
+
time = 0
|
|
656
|
+
comps = 0
|
|
657
|
+
for bdb, name in drep.d_cluster.cluster_utils.iteratre_clusters(Bdb, Cdb):
|
|
658
|
+
g = len(bdb['genome'].unique())
|
|
659
|
+
time += predict_fastani_time(g, cores)
|
|
660
|
+
comps += (g * g)
|
|
661
|
+
# Convert to minutes
|
|
662
|
+
time = time / 60
|
|
663
|
+
|
|
664
|
+
logging.info("Running {0} {1} comparisons- should take ~ {2:.1f} min".format(\
|
|
665
|
+
comps, algorithm, time))
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
def predict_fastani_time(genomes, cores=6):
|
|
669
|
+
'''
|
|
670
|
+
Forumula is f(x) = a*x**2 + b*x
|
|
671
|
+
|
|
672
|
+
http://localhost:8888/doc/tree/Other/OneOffs/dRep_performance_fastANI_timing_1.ipynb
|
|
673
|
+
'''
|
|
674
|
+
a = 0.018969442838018713
|
|
675
|
+
b = 14.604906958328073
|
|
676
|
+
|
|
677
|
+
genomes = int(genomes)
|
|
678
|
+
time_6cores = a * genomes ** 2 + b * genomes
|
|
679
|
+
|
|
680
|
+
time = time_6cores * (6 / cores)
|
|
681
|
+
|
|
682
|
+
return time
|
|
683
|
+
|
|
684
|
+
def _randomString(stringLength=10):
|
|
685
|
+
"""Generate a random string of fixed length """
|
|
686
|
+
letters = string.ascii_lowercase
|
|
687
|
+
return ''.join(random.choice(letters) for i in range(stringLength))
|