drep 4.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,687 @@
1
+ #!/usr/bin/env python3
2
+ '''
3
+ d_cluster - a subset of dRep
4
+
5
+ Clusters a list of genomes with both primary and secondary clustering
6
+ '''
7
+
8
+ import pandas as pd
9
+ import os
10
+ import logging
11
+ import random
12
+ import string
13
+ import numpy as np
14
+
15
+ import drep
16
+ import drep.d_cluster.cluster_utils
17
+ import drep.d_cluster.compare_utils
18
+ import drep.d_filter
19
+ import drep.d_bonus
20
+
21
+ # from drep.d_cluster import genomeChunk
22
+ import drep.d_cluster.controller
23
+ #from drep.d_cluster.controller import genomeChunk, genome_hierarchical_clustering, iteratre_clusters, cluster_hierarchical
24
+
25
+ # This is to make pandas shut up with it's warnings
26
+ pd.options.mode.chained_assignment = None
27
+
28
+ import drep.d_cluster.external
29
+ #from drep.d_cluster.external import parse_gani_file, parse_nsim_file, gen_nucmer_cmd, add_avani
30
+
31
+
32
+
33
+ def _cluster_Ndb(Ndb, id='primary_cluster', **kwargs):
34
+ '''
35
+ Cluster Ndb on id
36
+
37
+ Args:
38
+ Ndb: obvious
39
+ id: thing that defines subclusters in Ndb (defauls = 'primary_cluster')
40
+
41
+ Keyword arguments:
42
+ all: passed on to genome_hierarchical_clustering
43
+
44
+ Returns:
45
+ list: [Cdb, c2ret(cluster name -> clusering files)]
46
+ '''
47
+ Cdb = pd.DataFrame()
48
+ c2ret = {}
49
+ for name, ndb in Ndb.groupby(id):
50
+ cdb, cluster_ret = drep.d_cluster.cluster_utils.genome_hierarchical_clustering(ndb, cluster=name, **kwargs)
51
+ cdb[id] = name
52
+ Cdb = pd.concat([Cdb,cdb], ignore_index=True)
53
+ c2ret[name] = cluster_ret
54
+
55
+ return Cdb, c2ret
56
+
57
+
58
+ def make_linkage_Ndb(Ndb, **kwargs):
59
+ '''
60
+ Filter the Ndb in accordance with the kwargs. Average reciprical ANI values
61
+
62
+ Args:
63
+ Ndb: result of secondary clutsering
64
+
65
+ Keyword arguments:
66
+ cov_thresh: minimum coverage threshold (default = 0.5)
67
+
68
+ Return:
69
+ DataFrame: pivoted DataFrame ready for clustering
70
+ '''
71
+ d = Ndb.copy()
72
+ cov_thresh = float(kwargs.get('cov_thresh',0.5))
73
+
74
+ # Remove values without enough coverage
75
+ d.loc[d['alignment_coverage'] <= cov_thresh, 'ani'] = 0
76
+
77
+ # Make a linkagedb by averaging values and setting self-compare to 1
78
+ drep.d_cluster.external.add_avani(d)
79
+ d['dist'] = 1 - d['av_ani']
80
+ db = d.pivot(index="reference", columns="querry", values="dist")
81
+
82
+ return db
83
+
84
+
85
+ def estimate_time(comps, alg):
86
+ '''
87
+ Estimate time, in minutes, based on comparison algorithm and number of comparisons
88
+
89
+ Args:
90
+ comps: number of genomes comparisons to perform
91
+ alg: algorthm used
92
+
93
+ Return:
94
+ float: time to perfom comparison (in minutes)
95
+ '''
96
+ # Minutes per comparison, very roughly. This only drives a log message, so an
97
+ # unknown algorithm must never take the run down with it -- fall back to the
98
+ # fastest estimate rather than raising.
99
+ per_comparison = {
100
+ 'ANIn': .33,
101
+ 'gANI': .1,
102
+ 'goANI': .1,
103
+ 'ANImf': .5,
104
+ 'fastANI': 0.00667,
105
+ 'skani': 0.00667,
106
+ }
107
+ if alg not in per_comparison:
108
+ logging.debug(f"No time estimate available for {alg}; assuming a fast algorithm")
109
+ return comps * per_comparison.get(alg, 0.00667)
110
+
111
+
112
+
113
+
114
+ def _gen_cdb_from_fclust(fclust,names):
115
+ '''
116
+ Make Cdb from the result of scipy.cluster.hierarchy.fcluster
117
+
118
+ Args:
119
+ fclust: result of scipy.cluster.hierarchy.fcluster
120
+ names: list(db.columns) of the input dataframe
121
+
122
+ Returns:
123
+ DataFrame: Cdb
124
+ '''
125
+ Table={'cluster':[],'genome':[]}
126
+ for i, c in enumerate(fclust):
127
+ Table['cluster'].append(c)
128
+ Table['genome'].append(names[i])
129
+
130
+ return pd.DataFrame(Table)
131
+
132
+
133
+ def parse_mash_table(location):
134
+ try:
135
+ iniCols = ['genome1', 'genome2', 'dist', 'p', 'kmers']
136
+ uCols = ['genome1', 'genome2', 'dist']
137
+ dTypes = {'genome1': 'category', 'genome2': 'category', 'dist': np.float32}
138
+ Mdb = pd.read_csv(location, names=iniCols, usecols=uCols, dtype=dTypes, sep='\t')
139
+ except ValueError:
140
+ logging.error("You're using an incompatible version of Mash! I'll try and fix it, but I recommend upgrading Mash to v2.2 or downgrading to v1 if problems occur")
141
+ Mdb = pd.read_csv(location, sep='\t')
142
+
143
+ Mdb['genome1'] = Mdb['genome1'].apply(_get_genome_name_from_fasta).astype('category')
144
+ Mdb['genome2'] = Mdb['genome2'].apply(_get_genome_name_from_fasta).astype('category')
145
+ Mdb['similarity'] = 1 - Mdb['dist']
146
+
147
+ return Mdb
148
+
149
+
150
+ def merge_genome_chunks(mash_exe, genome_chunks, sketch_folder, MASH_folder):
151
+ all_file = os.path.join(MASH_folder, 'ALL.msh')
152
+ cmd = [mash_exe, 'paste', all_file] + [gc.all_file for gc in genome_chunks]
153
+
154
+ new_locs = []
155
+ new_names = []
156
+ for gc in genome_chunks:
157
+ new_locs += gc.genome_locations
158
+ new_names += gc.genome_names
159
+
160
+ new_gc = drep.d_cluster.compare_utils.genomeChunk(new_locs, 'all', sketch_folder, new_names, no_create=True)
161
+ new_gc.all_file = all_file
162
+
163
+ return cmd, new_gc
164
+
165
+
166
+ # def all_vs_all_MASH(Bdb, data_folder, **kwargs):
167
+ # """
168
+ # Run MASH pairwise within all samples in Bdb
169
+ #
170
+ # Args:
171
+ # Bdb: dataframe with genome, location
172
+ # data_folder: location to store output files
173
+ #
174
+ # Keyword Args:
175
+ # MASH_sketch: size of mash sketches
176
+ # dry: dont actually run anything
177
+ # processors: number of processors to multithread with
178
+ # mash_exe: location of mash excutible (will try and find with shutil if not provided)
179
+ # groupSize: max number of mash sketches to hold in each folder
180
+ # debug: if True, log all of the commands
181
+ # wd: if you want to log commands, you also need the wd
182
+ # """
183
+ #
184
+ # MASH_s = kwargs.get('MASH_sketch',1000)
185
+ # dry = kwargs.get('dry',False)
186
+ # # overwrite = kwargs.get('overwrite', False)
187
+ # p = kwargs.get('processors',6)
188
+ # groupSize = kwargs.get('groupSize', 1000)
189
+ #
190
+ # # set up logdir
191
+ # if ('wd' in kwargs) and (kwargs.get('debug', False) == True):
192
+ # logdir = kwargs.get('wd').get_dir('cmd_logs')
193
+ # else:
194
+ # logdir = False
195
+ #
196
+ # # Find mash
197
+ # mash_exe = kwargs.get('exe_loc', None)
198
+ # if mash_exe == None:
199
+ # mash_exe = drep.get_exe('mash')
200
+ #
201
+ # # Set up folders
202
+ # MASH_folder = os.path.join(data_folder, 'MASH_files/')
203
+ # if not os.path.exists(MASH_folder):
204
+ # os.makedirs(MASH_folder)
205
+ #
206
+ # sketch_folder = os.path.join(MASH_folder, 'sketches/')
207
+ # if not os.path.exists(sketch_folder):
208
+ # os.makedirs(sketch_folder)
209
+ #
210
+ # # Make chunks
211
+ # l2g = Bdb.set_index('location')['genome'].to_dict()
212
+ # locations = list(Bdb['location'].unique())
213
+ # chunks = [locations[x:x+groupSize] for x in range(0, len(locations), groupSize)]
214
+ #
215
+ # # Make the MASH sketches
216
+ # cmds = []
217
+ # chunk_folders = []
218
+ # for i, chunk in enumerate(chunks):
219
+ # chunk_folder = os.path.join(sketch_folder, "chunk_{0}".format(i))
220
+ # chunk_folders.append(chunk_folder)
221
+ # if not os.path.exists(chunk_folder):
222
+ # os.makedirs(chunk_folder)
223
+ # for fasta in chunk:
224
+ # genome = l2g[fasta]
225
+ # file = os.path.join(chunk_folder, genome)
226
+ # if not os.path.isfile(file + '.msh'):
227
+ # cmd = [mash_exe, 'sketch', fasta, '-s', str(MASH_s), '-o',
228
+ # file]
229
+ # cmds.append(cmd)
230
+ #
231
+ # if not dry:
232
+ # if len(cmds) > 0:
233
+ # drep.thread_cmds(cmds, logdir=logdir, t=int(p))
234
+ #
235
+ # # Combine MASH sketches within chunk
236
+ # cmds = []
237
+ # alls = []
238
+ # for chunk_folder in chunk_folders:
239
+ # all_file = os.path.join(chunk_folder, 'chunk_all.msh')
240
+ # cmd = [mash_exe, 'paste', all_file] \
241
+ # + glob.glob(os.path.join(chunk_folder, '*'))
242
+ # cmds.append(cmd)
243
+ # alls.append(all_file)
244
+ # if not dry:
245
+ # if len(cmds) > 0:
246
+ # drep.thread_cmds(cmds, logdir=logdir, t=int(p))
247
+ #
248
+ # # Combine MASH sketches of all chunks
249
+ # all_file = os.path.join(MASH_folder, 'ALL.msh')
250
+ # cmd = [mash_exe, 'paste', all_file] + alls
251
+ # drep.run_cmd(cmd, dry, shell=False, logdir=logdir)
252
+ #
253
+ # # Calculate distances
254
+ # cmd = [mash_exe, 'dist','-p', str(p), all_file, all_file, '>', MASH_folder
255
+ # + 'MASH_table.tsv']
256
+ # cmd = ' '.join(cmd)
257
+ # drep.run_cmd(cmd, dry, shell=True, logdir=logdir)
258
+ #
259
+ # # Make Mdb based on all genomes in the MASH folder
260
+ # file = MASH_folder + 'MASH_table.tsv'
261
+ #
262
+ # iniCols = ['genome1','genome2','dist','p','kmers']
263
+ # uCols = ['genome1','genome2','dist']
264
+ # dTypes = {'genome1':'category', 'genome2':'category', 'dist':np.float32}
265
+ # Mdb = pd.read_csv(file, names=iniCols, usecols=uCols, dtype=dTypes, sep='\t')
266
+ # Mdb['genome1'] = Mdb['genome1'].apply(_get_genome_name_from_fasta).astype('category')
267
+ # Mdb['genome2'] = Mdb['genome2'].apply(_get_genome_name_from_fasta).astype('category')
268
+ # Mdb['similarity'] = 1 - Mdb['dist']
269
+ #
270
+ # # Filter out those genomes that are in the MASH folder but shouldn't be in Mdb
271
+ # genomes = Bdb['genome'].unique()
272
+ # Mdb = Mdb[Mdb['genome1'].isin(genomes)]
273
+ # Mdb = Mdb[Mdb['genome2'].isin(genomes)]
274
+ #
275
+ # # Reorder categories to be correct
276
+ # for g in ['genome1', 'genome2']:
277
+ # Mdb[g] = Mdb[g].cat.remove_unused_categories()
278
+ # Mdb[g] = Mdb[g].cat.reorder_categories(sorted((Mdb[g].unique())), ordered=True)
279
+ #
280
+ # return Mdb
281
+
282
+ def _get_genome_name_from_fasta(fasta):
283
+ '''
284
+ Just do a os.path.basename command
285
+
286
+ Args:
287
+ fasta: location of .fasta file
288
+
289
+ Returns:
290
+ string: basename of .fasta files
291
+ '''
292
+ return str(os.path.basename(fasta))
293
+
294
+ def parse_delta(filename):
295
+ '''
296
+ Parse a .delta file from nucmer
297
+
298
+ Args:
299
+ filename: location of .delta file
300
+
301
+ Returns:
302
+ list: [alignment_length, similarity_errors]
303
+ '''
304
+
305
+ aln_length, sim_errors = 0, 0
306
+ for line in [l.strip().split() for l in open(filename, 'r').readlines()]:
307
+ if line[0] == 'NUCMER' or line[0].startswith('>'): # Skip headers
308
+ continue
309
+ # We only process lines with seven columns:
310
+ if len(line) == 7:
311
+ aln_length += abs(int(line[1]) - int(line[0]))
312
+ sim_errors += int(line[4])
313
+ return aln_length, sim_errors
314
+
315
+ def gen_gANI_cmd(file, g1, g2, exe):
316
+ '''
317
+ Generate a command to run gANI (ANIcalculator)
318
+
319
+ Will return [] if g1 == g2
320
+
321
+ Args:
322
+ file: output file name
323
+ g1: location of the genes of genome1
324
+ g2: location of the genes of genome2
325
+ exe: location of gANI executible
326
+
327
+ Returns:
328
+ list: command to run
329
+ '''
330
+ # Don't do self-comparions
331
+ if g1 == g2:
332
+ return []
333
+
334
+ else:
335
+ cmd = [exe,'-genome1fna',g1,'-genome2fna',g2,'-outfile',file,'-outdir',file + 'TEMP']
336
+ return cmd
337
+
338
+ def gen_goANI_cmd(file, g1, g2, exe):
339
+ '''
340
+ Generate a command to run goANI
341
+
342
+ Will return [] if g1 == g2
343
+
344
+ Args:
345
+ file: output file name
346
+ g1: location of the genes of genome1
347
+ g2: location of the genes of genome2
348
+ exe: location of nsimscan executible
349
+
350
+ Returns:
351
+ list: command to run
352
+ '''
353
+ # Don't do self-comparions
354
+ if g1 == g2:
355
+ return []
356
+
357
+ else:
358
+ cmd = [exe,'--om','TABX', '--rpq', '1', g1, g2, file]
359
+ return cmd
360
+
361
+ def process_deltafiles(deltafiles, org_lengths, logger=None, **kwargs):
362
+ '''
363
+ Parse a list of delta files into a pandas dataframe
364
+
365
+ Files NEED to be named in the format genome1_vs_genome2.delta, or genome1_vs_genome2.filtered.delta
366
+
367
+ Args:
368
+ deltafiles: list of .delta files
369
+ org_lengths: dictionary of genome to genome length
370
+
371
+ Keyword Arguments:
372
+ coverage_method: default is larger
373
+ logger: if not None, will log 0 errors
374
+
375
+ Returns:
376
+ DataFrame: information about the alignments
377
+ '''
378
+
379
+ Table = {'querry':[],'reference':[],'alignment_length':[],'similarity_errors':[],
380
+ 'ref_coverage':[],'querry_coverage':[],'ani':[], 'reference_length':[],
381
+ 'querry_length':[],'alignment_coverage':[]}
382
+
383
+ coverage_method = kwargs.get('coverage_method', 'larger')
384
+
385
+ for deltafile in deltafiles:
386
+ qname, sname = os.path.splitext(os.path.split(deltafile)[-1])[0].split('_vs_')
387
+ # Handle naming of "filtered"
388
+ if sname.endswith('.delta'):
389
+ sname = sname[:-6]
390
+
391
+ tot_length, tot_sim_error = parse_delta(deltafile)
392
+ if tot_length == 0 and logger is not None:
393
+ logging.info("Total alignment length reported in " +
394
+ "%s is zero!" % deltafile)
395
+ query_cover = float(tot_length) / org_lengths[qname]
396
+ sbjct_cover = float(tot_length) / org_lengths[sname]
397
+
398
+ try:
399
+ perc_id = 1 - float(tot_sim_error) / tot_length
400
+ except ZeroDivisionError:
401
+ #print("Alignment between {0} and {1} has 0 alignment!".format(qname,sname))
402
+ perc_id = 0 # set arbitrary value of zero identity
403
+
404
+
405
+ Table['querry'].append(qname)
406
+ Table['querry_length'].append(org_lengths[qname])
407
+ Table['reference'].append(sname)
408
+ Table['reference_length'].append(org_lengths[sname])
409
+ Table['alignment_length'].append(tot_length)
410
+ Table['similarity_errors'].append(tot_sim_error)
411
+ Table['ani'].append(perc_id)
412
+ Table['ref_coverage'].append(sbjct_cover)
413
+ Table['querry_coverage'].append(query_cover)
414
+
415
+ if coverage_method == 'total':
416
+ Table['alignment_coverage'].append((tot_length * 2)/(org_lengths[qname]\
417
+ + org_lengths[sname]))
418
+ elif coverage_method == 'larger':
419
+ Table['alignment_coverage'].append(max((tot_length/org_lengths[qname]),\
420
+ (tot_length/org_lengths[sname])))
421
+
422
+ dTypes = {'querry':'category','querry_length':np.int64, 'reference':'category',\
423
+ 'reference_length':np.int64, 'alignment_length':np.int64, 'similarity_errors':np.int64,\
424
+ 'ani':np.float32, 'ref_coverage':np.float32, 'querry_coverage':np.float32}
425
+ df = pd.DataFrame(Table)
426
+ for c, t in dTypes.items():
427
+ df[c] = df[c].astype(t)
428
+ return df
429
+
430
+ def process_goani_files(files):
431
+ '''
432
+ From a list of nsimscan files, return a parsed DataFrame like normal,
433
+ and a special one for calculating dn/ds
434
+
435
+ Args:
436
+ list: files
437
+
438
+ Returns:
439
+ DataFrame: Ndb
440
+ '''
441
+ Table = {'querry':[],'reference':[],'ani':[],'alignment_coverage':[]}
442
+ for file in files:
443
+ results = drep.d_cluster.external.parse_nsim_file(file)
444
+ results['reference'] = os.path.basename(file).split('_vs_')[0]
445
+ results['querry'] = os.path.basename(file).split('_vs_')[1][:-4]
446
+
447
+ # Add forward results
448
+ Table['reference'].append(results['reference'])
449
+ Table['querry'].append(results['querry'])
450
+ Table['ani'].append(float(results['ani'])/100)
451
+ Table['alignment_coverage'].append(results['af'])
452
+ #Table['alignment_coverage'].append(results['rq_coverage'])
453
+
454
+ # Add reverse results [they're the same]
455
+ # Table['reference'].append(results['querry'])
456
+ # Table['querry'].append(results['reference'])
457
+ # Table['ani'].append(float(results['ani'])/100)
458
+ # Table['alignment_coverage'].append(results['af'])
459
+ #Table['alignment_coverage'].append(results['qr_coverage'])
460
+
461
+ # Add self comparisons
462
+ for g in set(Table['reference']):
463
+ Table['reference'].append(g)
464
+ Table['querry'].append(g)
465
+ Table['ani'].append(1)
466
+ Table['alignment_coverage'].append(1)
467
+
468
+ Gdb = pd.DataFrame(Table)
469
+ return Gdb
470
+
471
+ def process_gani_files(files):
472
+ '''
473
+ From a list of gANI output files, return a parsed DataFrame
474
+
475
+ Args:
476
+ list: files
477
+
478
+ Returns:
479
+ DataFrame: Ndb
480
+ '''
481
+ Table = {'querry':[],'reference':[],'ani':[],'alignment_coverage':[]}
482
+ for file in files:
483
+ results = drep.d_cluster.external.parse_gani_file(file)
484
+
485
+ # calculate coverage using the "larger" method
486
+ cov = max(results['rq_coverage'], results['qr_coverage'])
487
+
488
+ # Add forward results
489
+ Table['reference'].append(results['reference'])
490
+ Table['querry'].append(results['querry'])
491
+ Table['ani'].append(float(results['rq_ani'])/100)
492
+ Table['alignment_coverage'].append(cov)
493
+ #Table['alignment_coverage'].append(results['rq_coverage'])
494
+
495
+ # Add reverse results [they're the same]
496
+ Table['reference'].append(results['querry'])
497
+ Table['querry'].append(results['reference'])
498
+ Table['ani'].append(float(results['qr_ani'])/100)
499
+ Table['alignment_coverage'].append(cov)
500
+ #Table['alignment_coverage'].append(results['qr_coverage'])
501
+
502
+ # Add self comparisons
503
+ for g in set(Table['reference']):
504
+ Table['reference'].append(g)
505
+ Table['querry'].append(g)
506
+ Table['ani'].append(1)
507
+ Table['alignment_coverage'].append(1)
508
+
509
+ Gdb = pd.DataFrame(Table)
510
+ return Gdb
511
+
512
+ def run_pairwise_ANIn(genome_list, ANIn_folder, **kwargs):
513
+ '''
514
+ Given a list of genomes and an output folder, compare all genomes using ANImf
515
+
516
+ Args:
517
+ genome_list: list of locations of genome files
518
+ ANIn_folder: folder to store the output of comparison
519
+
520
+ Keyword arguments:
521
+ processors: threads to use
522
+ debug: if true save extra output
523
+ wd: needed if debug is True
524
+ '''
525
+ p = kwargs.get('processors',6)
526
+ genomes = genome_list
527
+
528
+ # Make folder
529
+ if not os.path.exists(ANIn_folder):
530
+ os.makedirs(ANIn_folder)
531
+
532
+ # Gen commands
533
+ cmds = []
534
+ files = []
535
+ for g1 in genomes:
536
+ # Make it so each reference is it's own folder, to spread out .delta files
537
+ cur_folder = os.path.join(ANIn_folder, _get_genome_name_from_fasta(g1))
538
+ if not os.path.exists(cur_folder):
539
+ os.makedirs(cur_folder)
540
+
541
+ for g2 in genomes:
542
+ file_name = "{0}{1}_vs_{2}".format(ANIn_folder, \
543
+ _get_genome_name_from_fasta(g1),\
544
+ _get_genome_name_from_fasta(g2))
545
+ files.append(file_name)
546
+
547
+ # If the file doesn't already exist, add it to what needs to be run
548
+ if not os.path.isfile(file_name + '.delta'):
549
+ cmds.append(drep.d_cluster.external.gen_nucmer_cmd(file_name, g1, g2))
550
+
551
+ # Run commands
552
+ if len(cmds) > 0:
553
+ for c in cmds:
554
+ logging.debug(' '.join(c))
555
+
556
+ if ('wd' in kwargs) and (kwargs.get('debug', False)):
557
+ logdir = kwargs.get('wd').get_dir('cmd_logs')
558
+ else:
559
+ logdir = False
560
+ drep.thread_cmds(cmds, shell=False, logdir=logdir, t=int(p))
561
+
562
+ # Make dictionary of genome lengths
563
+ org_lengths = {}
564
+ for genome in genomes:
565
+ org_lengths[_get_genome_name_from_fasta(genome)] = \
566
+ drep.d_filter.calc_fasta_length(genome)
567
+
568
+
569
+
570
+ deltafiles = ["{0}.delta".format(file) for file in files]
571
+ df = process_deltafiles(deltafiles, org_lengths, **kwargs)
572
+
573
+ return df
574
+
575
+ def load_genomes(genome_list):
576
+ '''
577
+ Takes a list of genome locations, returns a pandas dataframe with the
578
+ locations and the basename of the genomes
579
+
580
+ Args:
581
+ genome_list: list of genome locations
582
+
583
+ Returns:
584
+ DataFrame: pandas dataframe with columns ['genome', 'location']
585
+ '''
586
+ assert type(genome_list) == type(list())
587
+
588
+ # Load from a text file
589
+ if len(genome_list) == 1:
590
+ logging.info('Loading genomes from a list')
591
+ try:
592
+ Table = {'genome':[],'location':[]}
593
+ with open(genome_list[0], 'r') as o:
594
+ for line in o.readlines():
595
+ genome = line.strip()
596
+ assert os.path.isfile(genome), "{0} is not a file".format(genome)
597
+ Table['genome'].append(os.path.basename(genome))
598
+ Table['location'].append(os.path.abspath(genome))
599
+ Bdb = pd.DataFrame(Table)
600
+ return Bdb
601
+ except:
602
+ logging.info('Nevermind! Ill try loading as a genome now')
603
+
604
+ Table = {'genome':[], 'location':[]}
605
+ for genome in genome_list:
606
+ assert os.path.isfile(genome), "{0} is not a file".format(genome)
607
+ Table['genome'].append(os.path.basename(genome))
608
+ Table['location'].append(os.path.abspath(genome))
609
+ Bdb = pd.DataFrame(Table)
610
+ return Bdb
611
+
612
+
613
+ def _gen_nomani_cdb(Cdb, **kwargs):
614
+ '''
615
+ Make a Cdb that looks like the Cdb from secondary clustering
616
+
617
+ Args:
618
+ Cdb: Result from Mash clustering
619
+
620
+ Returns:
621
+ DataFrame: Cdb
622
+ '''
623
+ c_method = kwargs.get('clusterAlg','single')
624
+ threshold = 1 - kwargs.get('P_ani',.9)
625
+
626
+ # Make Cdb look like Cdb
627
+ cdb = Cdb.copy()
628
+ cdb['secondary_cluster'] = ["{0}_0".format(i) for i in cdb['primary_cluster']]
629
+ cdb.rename(columns={'primary_cluster': 'primary_cluster'}, inplace=True)
630
+ cdb['threshold'] = threshold
631
+ cdb['cluster_method'] = c_method
632
+ cdb['comparison_algorithm'] = 'MASH'
633
+
634
+ return cdb
635
+
636
+ def _print_time_estimate(Bdb, Cdb, algorithm, cores):
637
+ '''
638
+ Print an estimate of how long genome comaprisons will take
639
+
640
+ Args:
641
+ Bdb: DataFrame with [genome, location]
642
+ Cdb: Clustering DataFrame
643
+ algorthm: algorithm to estimate time with
644
+ '''
645
+ if algorithm != 'fastANI':
646
+ comps = 0
647
+ for bdb, name in drep.d_cluster.cluster_utils.iteratre_clusters(Bdb, Cdb):
648
+ g = len(bdb['genome'].unique())
649
+ comps += (g * g)
650
+
651
+ time = estimate_time(comps, algorithm)
652
+ time = time / int(cores)
653
+
654
+ else:
655
+ time = 0
656
+ comps = 0
657
+ for bdb, name in drep.d_cluster.cluster_utils.iteratre_clusters(Bdb, Cdb):
658
+ g = len(bdb['genome'].unique())
659
+ time += predict_fastani_time(g, cores)
660
+ comps += (g * g)
661
+ # Convert to minutes
662
+ time = time / 60
663
+
664
+ logging.info("Running {0} {1} comparisons- should take ~ {2:.1f} min".format(\
665
+ comps, algorithm, time))
666
+
667
+
668
+ def predict_fastani_time(genomes, cores=6):
669
+ '''
670
+ Forumula is f(x) = a*x**2 + b*x
671
+
672
+ http://localhost:8888/doc/tree/Other/OneOffs/dRep_performance_fastANI_timing_1.ipynb
673
+ '''
674
+ a = 0.018969442838018713
675
+ b = 14.604906958328073
676
+
677
+ genomes = int(genomes)
678
+ time_6cores = a * genomes ** 2 + b * genomes
679
+
680
+ time = time_6cores * (6 / cores)
681
+
682
+ return time
683
+
684
+ def _randomString(stringLength=10):
685
+ """Generate a random string of fixed length """
686
+ letters = string.ascii_lowercase
687
+ return ''.join(random.choice(letters) for i in range(stringLength))