GenomeTreeTk 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- genometreetk/VERSION +136 -0
- genometreetk/__init__.py +22 -0
- genometreetk/arb.py +126 -0
- genometreetk/blast.py +85 -0
- genometreetk/bootstrap.py +373 -0
- genometreetk/cm_files/README +2 -0
- genometreetk/cm_files/lsu_archaea.cm +20824 -0
- genometreetk/cm_files/lsu_bacteria.cm +20327 -0
- genometreetk/cm_files/lsu_eukaryote.cm +23641 -0
- genometreetk/combine_support.py +151 -0
- genometreetk/common.py +104 -0
- genometreetk/derep_tree.py +236 -0
- genometreetk/fasttree.py +156 -0
- genometreetk/jackknife_markers.py +232 -0
- genometreetk/jackknife_taxa.py +136 -0
- genometreetk/main.py +667 -0
- genometreetk/msa_filter.py +499 -0
- genometreetk/phylogenetic_diversity.py +270 -0
- genometreetk/prune.py +99 -0
- genometreetk/reroot_tree.py +152 -0
- genometreetk/rna_workflow.py +834 -0
- genometreetk/tree_support.py +77 -0
- genometreetk-0.3.1.data/scripts/genometreetk +457 -0
- genometreetk-0.3.1.dist-info/METADATA +22 -0
- genometreetk-0.3.1.dist-info/RECORD +28 -0
- genometreetk-0.3.1.dist-info/WHEEL +5 -0
- genometreetk-0.3.1.dist-info/licenses/LICENSE +675 -0
- genometreetk-0.3.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
###############################################################################
|
|
2
|
+
# #
|
|
3
|
+
# This program is free software: you can redistribute it and/or modify #
|
|
4
|
+
# it under the terms of the GNU General Public License as published by #
|
|
5
|
+
# the Free Software Foundation, either version 3 of the License, or #
|
|
6
|
+
# (at your option) any later version. #
|
|
7
|
+
# #
|
|
8
|
+
# This program is distributed in the hope that it will be useful, #
|
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of #
|
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
|
|
11
|
+
# GNU General Public License for more details. #
|
|
12
|
+
# #
|
|
13
|
+
# You should have received a copy of the GNU General Public License #
|
|
14
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>. #
|
|
15
|
+
# #
|
|
16
|
+
###############################################################################
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
import logging
|
|
20
|
+
import multiprocessing as mp
|
|
21
|
+
import os
|
|
22
|
+
import random
|
|
23
|
+
import sys
|
|
24
|
+
from math import floor
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
|
|
27
|
+
from gtdblib.tree.bootstrap_merge import bootstrap_merge_replicates
|
|
28
|
+
from gtdblib.util.bio import seq_io
|
|
29
|
+
from gtdblib.util.shell.gtdbshutil import make_sure_path_exists
|
|
30
|
+
from tqdm import tqdm
|
|
31
|
+
|
|
32
|
+
from genometreetk.fasttree import FastTreeError, infer_trees
|
|
33
|
+
|
|
34
|
+
# State shared with forked worker processes (see JackknifeMarkers._create_replicates).
|
|
35
|
+
_SHARED = {}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _write_replicate_task(task):
|
|
39
|
+
"""Pool entry point for writing a jackknifed alignment."""
|
|
40
|
+
|
|
41
|
+
output_msa = task
|
|
42
|
+
JackknifeMarkers.jackknife_alignment(_SHARED['msa'],
|
|
43
|
+
_SHARED['perc_markers_to_keep'],
|
|
44
|
+
_SHARED['marker_lengths'],
|
|
45
|
+
output_msa)
|
|
46
|
+
return output_msa
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class JackknifeMarkers(object):
|
|
50
|
+
"""Assess robustness by jackkifing genes in alignment."""
|
|
51
|
+
|
|
52
|
+
def __init__(self, cpus):
|
|
53
|
+
"""Initialization.
|
|
54
|
+
|
|
55
|
+
Parameters
|
|
56
|
+
----------
|
|
57
|
+
cpus : int
|
|
58
|
+
Number of cpus to use.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
self.logger = logging.getLogger()
|
|
62
|
+
|
|
63
|
+
self.cpus = cpus
|
|
64
|
+
|
|
65
|
+
@staticmethod
|
|
66
|
+
def jackknife_alignment(msa, perc_markers_to_keep, marker_lengths, output_file):
|
|
67
|
+
"""Jackknife alignment to a subset of marker genes.
|
|
68
|
+
|
|
69
|
+
The marker_lengths must be specified in the order
|
|
70
|
+
in which genes were concatenated.
|
|
71
|
+
|
|
72
|
+
Parameters
|
|
73
|
+
----------
|
|
74
|
+
msa : d[seq_id] -> seq
|
|
75
|
+
Full multiple sequence alignment.
|
|
76
|
+
perc_markers_to_keep : float
|
|
77
|
+
Percentage of marker genes to keep in each replicate [0, 1].
|
|
78
|
+
marker_lengths : list
|
|
79
|
+
Length of each marker gene.
|
|
80
|
+
output_file : str
|
|
81
|
+
File to write bootstrapped alignment.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
markers_to_keep = random.sample(list(range(0, len(marker_lengths))),
|
|
85
|
+
int(floor(perc_markers_to_keep * len(marker_lengths))))
|
|
86
|
+
|
|
87
|
+
start_pos = [0]
|
|
88
|
+
for index, ml in enumerate(marker_lengths):
|
|
89
|
+
start_pos.append(start_pos[index] + ml)
|
|
90
|
+
|
|
91
|
+
# alignment columns of the kept markers, in their original order
|
|
92
|
+
segments = sorted((start_pos[i], start_pos[i] + marker_lengths[i]) for i in markers_to_keep)
|
|
93
|
+
|
|
94
|
+
tmp_file = output_file + '.tmp'
|
|
95
|
+
with open(tmp_file, 'w') as fout:
|
|
96
|
+
for seq_id, seq in msa.items():
|
|
97
|
+
fout.write('>' + seq_id + '\n')
|
|
98
|
+
fout.write(''.join(seq[start:end] for start, end in segments) + '\n')
|
|
99
|
+
os.replace(tmp_file, output_file)
|
|
100
|
+
|
|
101
|
+
def _create_replicates(self, msa_files):
|
|
102
|
+
"""Create jackknifed alignments in parallel."""
|
|
103
|
+
|
|
104
|
+
_SHARED.update(msa=self.msa,
|
|
105
|
+
perc_markers_to_keep=self.perc_markers_to_keep,
|
|
106
|
+
marker_lengths=self.marker_lengths)
|
|
107
|
+
try:
|
|
108
|
+
if self.cpus > 1 and len(msa_files) > 1:
|
|
109
|
+
# fork is used so workers share the MSA in memory
|
|
110
|
+
ctx = mp.get_context('fork')
|
|
111
|
+
with ctx.Pool(min(self.cpus, len(msa_files))) as pool:
|
|
112
|
+
for _ in tqdm(pool.imap_unordered(_write_replicate_task, msa_files),
|
|
113
|
+
total=len(msa_files), unit=' replicates'):
|
|
114
|
+
pass
|
|
115
|
+
else:
|
|
116
|
+
for msa_file in tqdm(msa_files, unit=' replicates'):
|
|
117
|
+
_write_replicate_task(msa_file)
|
|
118
|
+
finally:
|
|
119
|
+
_SHARED.clear()
|
|
120
|
+
|
|
121
|
+
def run(self, input_tree,
|
|
122
|
+
msa_file,
|
|
123
|
+
marker_info_file,
|
|
124
|
+
mask_file,
|
|
125
|
+
perc_markers_to_keep,
|
|
126
|
+
num_replicates,
|
|
127
|
+
model,
|
|
128
|
+
jk_dir,
|
|
129
|
+
output_dir):
|
|
130
|
+
"""Jackknife marker genes.
|
|
131
|
+
|
|
132
|
+
Marker file should have the format:
|
|
133
|
+
<marker id>\t<marker name>\t<marker desc>\t<length>\n
|
|
134
|
+
|
|
135
|
+
Parameters
|
|
136
|
+
----------
|
|
137
|
+
input_tree : str
|
|
138
|
+
Tree inferred with all data.
|
|
139
|
+
msa_file : str
|
|
140
|
+
File containing multiple sequence alignment for all taxa.
|
|
141
|
+
marker_info_file : str
|
|
142
|
+
File indicating database id, HMM name, description and length of each marker in the alignment.
|
|
143
|
+
mask_file : str
|
|
144
|
+
File indicating masking of multiple sequence alignment.
|
|
145
|
+
perc_markers_to_keep : float [0, 1]
|
|
146
|
+
Percentage of marker genes to keep in each replicate.
|
|
147
|
+
num_replicates : int
|
|
148
|
+
Number of replicates to perform.
|
|
149
|
+
model : str
|
|
150
|
+
Desired model of evolution.
|
|
151
|
+
jk_dir : str
|
|
152
|
+
Directory with pre-computed jackknife trees.
|
|
153
|
+
output_dir : str
|
|
154
|
+
Output directory for jackkife trees.
|
|
155
|
+
"""
|
|
156
|
+
|
|
157
|
+
assert(model in ['wag', 'jtt'])
|
|
158
|
+
|
|
159
|
+
self.model = model
|
|
160
|
+
self.perc_markers_to_keep = perc_markers_to_keep
|
|
161
|
+
|
|
162
|
+
rep_tree_files = []
|
|
163
|
+
if not jk_dir:
|
|
164
|
+
self.replicate_dir = os.path.join(output_dir, 'replicates')
|
|
165
|
+
make_sure_path_exists(self.replicate_dir)
|
|
166
|
+
|
|
167
|
+
# determine length of each marker gene in alignment
|
|
168
|
+
marker_lengths = []
|
|
169
|
+
total_len = 0
|
|
170
|
+
with open(marker_info_file) as f:
|
|
171
|
+
f.readline()
|
|
172
|
+
for line in f:
|
|
173
|
+
line_split = line.split('\t')
|
|
174
|
+
ml = int(line_split[3])
|
|
175
|
+
marker_lengths.append(ml)
|
|
176
|
+
total_len += ml
|
|
177
|
+
|
|
178
|
+
self.logger.info('Concatenated length of markers: %d' % total_len)
|
|
179
|
+
|
|
180
|
+
# read mask
|
|
181
|
+
with open(mask_file) as f:
|
|
182
|
+
mask = f.readline().strip()
|
|
183
|
+
start = 0
|
|
184
|
+
self.marker_lengths = []
|
|
185
|
+
total_mask_len = 0
|
|
186
|
+
for ml in marker_lengths:
|
|
187
|
+
end = start + ml
|
|
188
|
+
zeros = mask[start:end].count('0')
|
|
189
|
+
start = end
|
|
190
|
+
|
|
191
|
+
self.marker_lengths.append(ml - zeros)
|
|
192
|
+
total_mask_len += ml - zeros
|
|
193
|
+
|
|
194
|
+
self.logger.info('Concatenated length of filtered MSA: %d' % total_mask_len)
|
|
195
|
+
|
|
196
|
+
# read full multiple sequence alignment
|
|
197
|
+
self.msa = seq_io.read(msa_file)
|
|
198
|
+
|
|
199
|
+
if len(list(self.msa.values())[0]) != total_mask_len:
|
|
200
|
+
self.logger.error('Length of MSA does not meet length of mask.')
|
|
201
|
+
sys.exit(1)
|
|
202
|
+
|
|
203
|
+
# create jackknifed alignments and infer trees
|
|
204
|
+
msa_files = [os.path.join(self.replicate_dir, f'jk_markers.msa.{i}.faa') for i in range(num_replicates)]
|
|
205
|
+
rep_tree_files = [os.path.join(self.replicate_dir, f'jk_markers.tree.{i}.tre') for i in range(num_replicates)]
|
|
206
|
+
|
|
207
|
+
self.logger.info('Calculating jackknife marker replicates:')
|
|
208
|
+
self._create_replicates(msa_files)
|
|
209
|
+
self.msa = None
|
|
210
|
+
|
|
211
|
+
self.logger.info(f'Inferring {num_replicates:,} jackknife trees using {self.cpus:,} CPUs:')
|
|
212
|
+
try:
|
|
213
|
+
infer_trees(msa_files, rep_tree_files, cpus=self.cpus, base_type='prot', model=self.model)
|
|
214
|
+
except FastTreeError as e:
|
|
215
|
+
self.logger.error(str(e))
|
|
216
|
+
sys.exit(1)
|
|
217
|
+
else:
|
|
218
|
+
for f in sorted(os.listdir(jk_dir)):
|
|
219
|
+
if f.endswith('.tree') or f.endswith('.tre'):
|
|
220
|
+
rep_tree_files.append(os.path.join(jk_dir, f))
|
|
221
|
+
|
|
222
|
+
self.logger.info('Read %d jackknife replicates.' % len(rep_tree_files))
|
|
223
|
+
if not rep_tree_files:
|
|
224
|
+
self.logger.error(f'No jackknife trees found in: {jk_dir}')
|
|
225
|
+
sys.exit(1)
|
|
226
|
+
|
|
227
|
+
# calculate support
|
|
228
|
+
self.logger.info(f'Calculating support from {len(rep_tree_files):,} replicates.')
|
|
229
|
+
output_tree = os.path.join(output_dir, Path(input_tree).stem + '.jk_markers.tree')
|
|
230
|
+
bootstrap_merge_replicates(input_tree, output_tree, rep_tree_files, self.cpus)
|
|
231
|
+
|
|
232
|
+
return output_tree
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
###############################################################################
|
|
2
|
+
# #
|
|
3
|
+
# This program is free software: you can redistribute it and/or modify #
|
|
4
|
+
# it under the terms of the GNU General Public License as published by #
|
|
5
|
+
# the Free Software Foundation, either version 3 of the License, or #
|
|
6
|
+
# (at your option) any later version. #
|
|
7
|
+
# #
|
|
8
|
+
# This program is distributed in the hope that it will be useful, #
|
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of #
|
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
|
|
11
|
+
# GNU General Public License for more details. #
|
|
12
|
+
# #
|
|
13
|
+
# You should have received a copy of the GNU General Public License #
|
|
14
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>. #
|
|
15
|
+
# #
|
|
16
|
+
###############################################################################
|
|
17
|
+
|
|
18
|
+
import os
|
|
19
|
+
import logging
|
|
20
|
+
import random
|
|
21
|
+
from math import floor
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
from gtdblib.util.bio import seq_io
|
|
25
|
+
from gtdblib.util.shell.gtdbshutil import make_sure_path_exists
|
|
26
|
+
|
|
27
|
+
from genometreetk.fasttree import infer_trees
|
|
28
|
+
|
|
29
|
+
from genometreetk.tree_support import TreeSupport
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class JackknifeTaxa(object):
|
|
33
|
+
"""Assess robustness by jackknifing taxa in alignment."""
|
|
34
|
+
|
|
35
|
+
def __init__(self, cpus):
|
|
36
|
+
"""Initialization.
|
|
37
|
+
|
|
38
|
+
Parameters
|
|
39
|
+
----------
|
|
40
|
+
cpus : int
|
|
41
|
+
Number of cpus to use.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
self.logger = logging.getLogger()
|
|
45
|
+
|
|
46
|
+
self.cpus = cpus
|
|
47
|
+
|
|
48
|
+
def _create_replicates(self, num_replicates):
|
|
49
|
+
"""Create jackknifed alignments and infer a tree for each."""
|
|
50
|
+
|
|
51
|
+
msa_files = [os.path.join(self.replicate_dir, f'jk_taxa.msa.{i}.fna') for i in range(num_replicates)]
|
|
52
|
+
tree_files = [os.path.join(self.replicate_dir, f'jk_taxa.tree.{i}.tre') for i in range(num_replicates)]
|
|
53
|
+
for msa_file in msa_files:
|
|
54
|
+
self.jackknife_taxa(self.msa, self.perc_taxa_to_keep, self.outgroup_ids, msa_file)
|
|
55
|
+
|
|
56
|
+
infer_trees(msa_files, tree_files, cpus=self.cpus, base_type='prot', model=self.model)
|
|
57
|
+
|
|
58
|
+
def jackknife_taxa(self, msa, perc_taxa_to_keep, outgroup_ids, output_file):
|
|
59
|
+
"""Jackknife alignment to a subset of taxa.
|
|
60
|
+
|
|
61
|
+
Parameters
|
|
62
|
+
----------
|
|
63
|
+
msa : d[seq_id] -> seq
|
|
64
|
+
Full multiple sequence alignment.
|
|
65
|
+
perc_taxa_to_keep : float
|
|
66
|
+
Percentage of marker genes to keep in each replicate.
|
|
67
|
+
outgroup_ids : set
|
|
68
|
+
Labels of outgroup taxa.
|
|
69
|
+
output_file : str
|
|
70
|
+
File to write bootstrapped alignment.
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
# randomly select ingroup taxa
|
|
74
|
+
ingroup_taxa = set(msa.keys()) - outgroup_ids
|
|
75
|
+
taxa_to_keep = random.sample(sorted(ingroup_taxa), int(floor(len(ingroup_taxa) * perc_taxa_to_keep)))
|
|
76
|
+
|
|
77
|
+
taxa_to_keep = set(taxa_to_keep).union(outgroup_ids)
|
|
78
|
+
|
|
79
|
+
fout = open(output_file, 'w')
|
|
80
|
+
for seq_id, seq in msa.items():
|
|
81
|
+
if seq_id in taxa_to_keep:
|
|
82
|
+
fout.write('>' + seq_id + '\n')
|
|
83
|
+
fout.write(seq + '\n')
|
|
84
|
+
fout.close()
|
|
85
|
+
|
|
86
|
+
def run(self, input_tree, msa_file, outgroup_file, perc_taxa_to_keep, num_replicates, model, output_dir):
|
|
87
|
+
"""Jackknife taxa.
|
|
88
|
+
|
|
89
|
+
Parameters
|
|
90
|
+
----------
|
|
91
|
+
input_tree : str
|
|
92
|
+
Tree inferred with all data.
|
|
93
|
+
msa_file : str
|
|
94
|
+
File containing multiple sequence alignment for all taxa.
|
|
95
|
+
outgroup_file : str
|
|
96
|
+
File indicating labels of outgroup taxa.
|
|
97
|
+
perc_taxa_to_keep : float
|
|
98
|
+
Percentage of taxa to keep in each replicate.
|
|
99
|
+
num_replicates : int
|
|
100
|
+
Number of replicates to perform.
|
|
101
|
+
model : str
|
|
102
|
+
Desired model of evolution.
|
|
103
|
+
output_dir : str
|
|
104
|
+
input_tree directory for bootstrap trees.
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
assert(model in ['wag', 'jtt'])
|
|
108
|
+
|
|
109
|
+
self.perc_taxa_to_keep = perc_taxa_to_keep
|
|
110
|
+
self.model = model
|
|
111
|
+
self.replicate_dir = os.path.join(output_dir, 'replicates')
|
|
112
|
+
make_sure_path_exists(self.replicate_dir)
|
|
113
|
+
|
|
114
|
+
# read outgroup taxa
|
|
115
|
+
self.outgroup_ids = set()
|
|
116
|
+
if outgroup_file:
|
|
117
|
+
for line in open(outgroup_file):
|
|
118
|
+
self.outgroup_ids.add(line.strip())
|
|
119
|
+
|
|
120
|
+
# read full multiple sequence alignment
|
|
121
|
+
self.msa = seq_io.read(msa_file)
|
|
122
|
+
|
|
123
|
+
# calculate replicates
|
|
124
|
+
#***self.logger.info('Calculating jackknife taxa replicates:')
|
|
125
|
+
#***self._create_replicates(num_replicates)
|
|
126
|
+
|
|
127
|
+
# calculate support
|
|
128
|
+
rep_tree_files = []
|
|
129
|
+
for rep_index in range(num_replicates):
|
|
130
|
+
rep_tree_files.append(os.path.join(self.replicate_dir, 'jk_taxa.tree.' + str(rep_index) + '.tre'))
|
|
131
|
+
|
|
132
|
+
tree_support = TreeSupport()
|
|
133
|
+
output_tree = os.path.join(output_dir, Path(input_tree).stem + '.jk_taxa.tree')
|
|
134
|
+
tree_support.subset_taxa(input_tree, rep_tree_files, output_tree)
|
|
135
|
+
|
|
136
|
+
return output_tree
|