GenomeTreeTk 0.3.1__py3-none-any.whl

genometreetk/VERSION

@@ -0,0 +1,136 @@
+0.3.1
+- requires gtdblib >= 1.12.1
+- messages from gtdblib use the GenomeTreeTk log format, are written to genometreetk.log, and are suppressed by --silent
+- errors reported by gtdblib (e.g. a taxon without a rank prefix) are logged with exit code 1 instead of a traceback
+
+0.3.0
+- replaced biolib with gtdblib (>= 1.12.0); packaging now uses setuptools and requires Python >= 3.7
+- bootstrap: MSA replicates are created from a shared numpy matrix (about 7x faster and 4x less memory for 200,000 x 5,000 MSAs)
+- bootstrap: added --seed; replicates are identical for any number of CPUs
+- bootstrap: reruns safely resume interrupted runs; FastTree failures are reported instead of ignored
+- bootstrap: fixed --boot_dir with MSA replicates; --msa_only no longer requires FastTree
+- bootstrap and jk_markers: support values are calculated with gtdblib, which keeps the rooting and taxon labels of the input tree and writes whole-number support
+- jk_markers: FastTree failures are reported instead of ignored
+- combine: handles nodes without support values (e.g. the root) and decimal support values
+- errors now exit with a non-zero code
+- fixed pull writing invalid taxonomy strings for leaves without labelled ancestors
+- fixed derep_tree (required --msa_file and failed on Python 3) and arb_records (failed on Python 3)
+- removed unused modules (infer_workflow, markers, ncbi, aai)
+- added end-to-end tests of the commands in tests/
+
+0.2.3
+- added sh_uf_hack command
+
+0.2.2
+- fixed bug with filtered MSA columns being written out in a random order within each marker
+- now generating final_mask.txt which is the mask after filtering and subsampling
+
+0.2.1
+- modified bootstrap method to support use of FastTreeMP with multiple CPUs
+
+0.2.0
+- added msa_filter method
+
+0.1.9
+- added support to create MSA replicates without inferring replicate trees
+- added support to infer trees from MSA replicates if they do not already exist
+
+0.1.8
+- better support for doing bootstrap trees across multiple servers
+
+0.1.7
+- updated propagate command to work with new Gxyz style genome IDs
+- updated 'bootstrap' command to not require input tree
+
+0.1.6
+- fixed more merging issues
+
+0.1.5
+- fixed mering issue
+
+0.1.4
+- added prune method
+
+0.1.3
+- modified main.py to have Python 3 print statements
+
+0.1.2
+- updated shebang to Python 3
+
+0.1.1
+- added CM data files to repository
+
+0.1.0
+- initial migration to Python 3
+
+0.0.55
+- added rm_support
+- bootstrap command will not look for files with the extension '.treefile'
+
+0.0.54
+- modified PD calculations when ingroup does not encompass root node; this
+  makes PD a rooted tree calculation, but this is probably best for most
+  situations
+- simplified interface to PD calculations to remove notion of representative genomes
+
+0.0.53
+- better handling of support values when 'stripping' a tree
+
+0.0.52
+- modified cluster_de_novo to ensure all genomes as assigned to the closest type genome
+- fixed synonym table: species were being indicated as synonyms of multiple species (e.g. Shigella boydii)
+
+0.0.51
+- added --rnd_type_genome option for evaluating impact of type genome selection on de novo clusters
+
+0.0.50
+- added --gamma option to bootstrap command so it isn't automatically used
+
+0.0.49
+- used for GTDB r89
+- added QC exception list to qc_genomes command
+- added species classification exception list to qc_genomes command
+- added species classification exception list to select_type_genomes command
+- added species classification exception list to cluster_named_types command
+- added GTDB type genomes exception list to select_type_genomes command
+- added species type genome radius information to file indicating species clusters
+- simplified interface to cluster_user as species ANI radius information is now in cluster file
+- stripped 'Candidatus' from all species names
+
+0.0.48
+- added type species of genus information for resolving priority of species with same publication date
+- changed {manual_curation} to {unresolved}
+- changed determination of genus name for de novo species clusters to consider most common genus name in cluster
+- added checks to ensure genomes marked as assembled from type material or NCBI representative genomes as not actually a subspecies
+
+0.0.47
+- initial implementation of filtering of percentage of bac120 or ar122 marker genes
+
+0.0.46
+- initial type genome workflow for GTDB r89
+
+0.0.45
+- improved dereplication workflow for GTDB r89
+
+0.0.44
+- add method to dump 5S, 16S, and 23S sequences to file
+
+0.0.43
+- improvements to code for assigning genomes to canonical genome set 
+
+0.0.42
+- initial implementation of code to assign genomes to canonical genome set 
+
+0.0.41
+- fixed bug with parsing contig length from SSU file
+
+0.0.40
+- fixed issue with rerooting trees with bootstraps between [0, 1]
+
+0.0.39
+- added GTR model when inferring SSU or LSU trees
+
+0.0.38
+- new method for selecting GTDB representatives
+
+0.0.37

genometreetk/__init__.py

@@ -0,0 +1,22 @@
+###############################################################################
+#                                                                             #
+#    This program is free software: you can redistribute it and/or modify     #
+#    it under the terms of the GNU General Public License as published by     #
+#    the Free Software Foundation, either version 3 of the License, or        #
+#    (at your option) any later version.                                      #
+#                                                                             #
+#    This program is distributed in the hope that it will be useful,          #
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of           #
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            #
+#    GNU General Public License for more details.                             #
+#                                                                             #
+#    You should have received a copy of the GNU General Public License        #
+#    along with this program. If not, see <http://www.gnu.org/licenses/>.     #
+#                                                                             #
+###############################################################################
+
+import os
+
+# Set the module version.
+with open(os.path.join(__path__[0], 'VERSION'), 'r') as f:
+    __version__ = f.readline().strip()

genometreetk/arb.py

@@ -0,0 +1,126 @@
+###############################################################################
+#                                                                             #
+#    This program is free software: you can redistribute it and/or modify     #
+#    it under the terms of the GNU General Public License as published by     #
+#    the Free Software Foundation, either version 3 of the License, or        #
+#    (at your option) any later version.                                      #
+#                                                                             #
+#    This program is distributed in the hope that it will be useful,          #
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of           #
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            #
+#    GNU General Public License for more details.                             #
+#                                                                             #
+#    You should have received a copy of the GNU General Public License        #
+#    along with this program. If not, see <http://www.gnu.org/licenses/>.     #
+#                                                                             #
+###############################################################################
+
+import os
+import sys
+import csv
+import logging
+
+import gtdblib.util.bio.seq_io as seq_io
+from gtdblib.taxonomy.taxonomy import read_taxonomy
+
+
+class Arb(object):
+    """Methods for producing files for ARB."""
+
+    def __init__(self):
+        """Initialization."""
+
+        self.logger = logging.getLogger()
+        
+    def _record(self, fout, 
+                    genome_id,
+                    metadata_fields, 
+                    metadata_values,
+                    aligned_seq):
+        """Write out ARB record for genome."""
+
+        fout.write("BEGIN\n")
+        fout.write("db_name=%s\n" % genome_id)
+        for col_header, value in zip(metadata_fields, metadata_values):
+            # replace equal signs as these are incompatible with the ARB parser
+            if value:
+                value = value.replace('=', '/')
+
+            fout.write("%s=%s\n" % (col_header, value))
+        
+        fout.write("warning=\n")
+        fout.write("aligned_seq=%s\n" % (aligned_seq))
+        fout.write("END\n\n")
+
+    def create_records(self, metadata_file, msa_file, taxonomy_file, genome_list, output_file):
+        """Create ARB records from GTDB metadata."""
+        
+        seqs = {}
+        if msa_file:
+            seqs = seq_io.read(msa_file)
+        
+        taxonomy = {}
+        if taxonomy_file:
+            taxonomy = read_taxonomy(taxonomy_file)
+            
+        genomes_to_keep = set()
+        if genome_list:
+            for line in open(genome_list):
+                genomes_to_keep.add(line.strip())
+                
+        fout = open(output_file, 'w')
+        
+        delimiter = ','
+        if metadata_file.endswith('.tsv'):
+            delimiter = '\t'
+        
+        header = True
+        for row in csv.reader(open(metadata_file, newline=''), delimiter=delimiter):
+            if header:
+                fields = [f.lower().replace(' ', '_').replace('-', '_') for f in row[1:]]
+                if taxonomy:
+                    fields.append('gtdb_taxonomy')
+                header = False
+            else:
+                genome_id = row[0]
+                values = row[1:]
+                if taxonomy:
+                    values.append('; '.join(taxonomy[genome_id]))
+                aligned_seq = seqs.get(genome_id, '')
+                
+                if not genomes_to_keep or genome_id in genomes_to_keep:
+                    self._record(fout, genome_id, fields, values, aligned_seq)
+
+        fout.close()
+        
+    def write(self, hashes, io):
+        """Write data to a GreenGenes formatted files.
+
+        Parameters
+        ----------
+        hashes : list of dict[feature] -> value
+            GreenGenes style metadata.
+        io : output stream
+            Handle to output stream.
+        """
+        is_first = True
+        for dahash in hashes:
+            if is_first:
+                is_first = False
+            else:
+                io.write('\n')
+
+            io.write('BEGIN\n')
+            for key in sorted(dahash.keys()):
+                if key != 'warning' and key != 'aligned_seq':
+                    io.write('='.join([key, dahash[key]]) + '\n')
+
+            # the warning field must be the second to last as it is used
+            # to indicates that the aligned sequence is to follow
+            io.write('='.join(['warning', dahash.get('warning', '')]) + '\n')
+
+            # the aligned sequence must be the last field
+            io.write('='.join(['aligned_seq', dahash.get('aligned_seq', '')]) + '\n')
+
+            io.write('END\n')
+            

genometreetk/blast.py

@@ -0,0 +1,85 @@
+###############################################################################
+#                                                                             #
+#    This program is free software: you can redistribute it and/or modify     #
+#    it under the terms of the GNU General Public License as published by     #
+#    the Free Software Foundation, either version 3 of the License, or        #
+#    (at your option) any later version.                                      #
+#                                                                             #
+#    This program is distributed in the hope that it will be useful,          #
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of           #
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            #
+#    GNU General Public License for more details.                             #
+#                                                                             #
+#    You should have received a copy of the GNU General Public License        #
+#    along with this program. If not, see <http://www.gnu.org/licenses/>.     #
+#                                                                             #
+###############################################################################
+
+"""Minimal BLAST wrapper used to identify rRNA sequences with incongruent taxonomy."""
+
+import subprocess
+from collections import namedtuple
+
+from gtdblib.util.shell.execute import check_dependencies
+
+# fields written by blastn with BLAST_CUSTOM_FMT
+BLAST_CUSTOM_FMT = '6 qseqid qlen sseqid stitle slen length pident evalue bitscore'
+BlastHit = namedtuple('BlastHit', ['query_id',
+                                   'query_len',
+                                   'subject_id',
+                                   'subject_annotation',
+                                   'subject_len',
+                                   'alignment_len',
+                                   'perc_identity',
+                                   'evalue',
+                                   'bitscore'])
+
+
+class BlastError(Exception):
+    pass
+
+
+def _run(cmd):
+    proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+    if proc.returncode != 0:
+        raise BlastError('{} failed with exit code {}: {}'.format(
+            cmd[0], proc.returncode, proc.stderr.strip()))
+
+
+def create_blastn_db(fasta_file, db_file):
+    """Create nucleotide BLAST database."""
+
+    check_dependencies(['makeblastdb'])
+    _run(['makeblastdb', '-dbtype', 'nucl', '-in', fasta_file, '-out', db_file])
+
+
+def blastn(query_seqs, nucl_db, output_file, cpus=1, evalue=1e-3, max_matches=500, task='megablast'):
+    """Search query sequences against a nucleotide database, writing hits in BLAST_CUSTOM_FMT."""
+
+    check_dependencies(['blastn'])
+    _run(['blastn',
+          '-num_threads', str(cpus),
+          '-query', query_seqs,
+          '-db', nucl_db,
+          '-out', output_file,
+          '-evalue', str(evalue),
+          '-max_target_seqs', str(max_matches),
+          '-task', task,
+          '-outfmt', BLAST_CUSTOM_FMT])
+
+
+def read_hits(table):
+    """Read hits from a table written by blastn()."""
+
+    with open(table) as f:
+        for line in f:
+            tokens = line.rstrip('\n').split('\t')
+            yield BlastHit(query_id=tokens[0],
+                           query_len=int(tokens[1]),
+                           subject_id=tokens[2],
+                           subject_annotation=tokens[3],
+                           subject_len=int(tokens[4]),
+                           alignment_len=int(tokens[5]),
+                           perc_identity=float(tokens[6]),
+                           evalue=float(tokens[7]),
+                           bitscore=float(tokens[8]))