GenomeTreeTk 0.3.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,232 @@
1
+ ###############################################################################
2
+ # #
3
+ # This program is free software: you can redistribute it and/or modify #
4
+ # it under the terms of the GNU General Public License as published by #
5
+ # the Free Software Foundation, either version 3 of the License, or #
6
+ # (at your option) any later version. #
7
+ # #
8
+ # This program is distributed in the hope that it will be useful, #
9
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of #
10
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
11
+ # GNU General Public License for more details. #
12
+ # #
13
+ # You should have received a copy of the GNU General Public License #
14
+ # along with this program. If not, see <http://www.gnu.org/licenses/>. #
15
+ # #
16
+ ###############################################################################
17
+
18
+
19
+ import logging
20
+ import multiprocessing as mp
21
+ import os
22
+ import random
23
+ import sys
24
+ from math import floor
25
+ from pathlib import Path
26
+
27
+ from gtdblib.tree.bootstrap_merge import bootstrap_merge_replicates
28
+ from gtdblib.util.bio import seq_io
29
+ from gtdblib.util.shell.gtdbshutil import make_sure_path_exists
30
+ from tqdm import tqdm
31
+
32
+ from genometreetk.fasttree import FastTreeError, infer_trees
33
+
34
+ # State shared with forked worker processes (see JackknifeMarkers._create_replicates).
35
+ _SHARED = {}
36
+
37
+
38
+ def _write_replicate_task(task):
39
+ """Pool entry point for writing a jackknifed alignment."""
40
+
41
+ output_msa = task
42
+ JackknifeMarkers.jackknife_alignment(_SHARED['msa'],
43
+ _SHARED['perc_markers_to_keep'],
44
+ _SHARED['marker_lengths'],
45
+ output_msa)
46
+ return output_msa
47
+
48
+
49
+ class JackknifeMarkers(object):
50
+ """Assess robustness by jackkifing genes in alignment."""
51
+
52
+ def __init__(self, cpus):
53
+ """Initialization.
54
+
55
+ Parameters
56
+ ----------
57
+ cpus : int
58
+ Number of cpus to use.
59
+ """
60
+
61
+ self.logger = logging.getLogger()
62
+
63
+ self.cpus = cpus
64
+
65
+ @staticmethod
66
+ def jackknife_alignment(msa, perc_markers_to_keep, marker_lengths, output_file):
67
+ """Jackknife alignment to a subset of marker genes.
68
+
69
+ The marker_lengths must be specified in the order
70
+ in which genes were concatenated.
71
+
72
+ Parameters
73
+ ----------
74
+ msa : d[seq_id] -> seq
75
+ Full multiple sequence alignment.
76
+ perc_markers_to_keep : float
77
+ Percentage of marker genes to keep in each replicate [0, 1].
78
+ marker_lengths : list
79
+ Length of each marker gene.
80
+ output_file : str
81
+ File to write bootstrapped alignment.
82
+ """
83
+
84
+ markers_to_keep = random.sample(list(range(0, len(marker_lengths))),
85
+ int(floor(perc_markers_to_keep * len(marker_lengths))))
86
+
87
+ start_pos = [0]
88
+ for index, ml in enumerate(marker_lengths):
89
+ start_pos.append(start_pos[index] + ml)
90
+
91
+ # alignment columns of the kept markers, in their original order
92
+ segments = sorted((start_pos[i], start_pos[i] + marker_lengths[i]) for i in markers_to_keep)
93
+
94
+ tmp_file = output_file + '.tmp'
95
+ with open(tmp_file, 'w') as fout:
96
+ for seq_id, seq in msa.items():
97
+ fout.write('>' + seq_id + '\n')
98
+ fout.write(''.join(seq[start:end] for start, end in segments) + '\n')
99
+ os.replace(tmp_file, output_file)
100
+
101
+ def _create_replicates(self, msa_files):
102
+ """Create jackknifed alignments in parallel."""
103
+
104
+ _SHARED.update(msa=self.msa,
105
+ perc_markers_to_keep=self.perc_markers_to_keep,
106
+ marker_lengths=self.marker_lengths)
107
+ try:
108
+ if self.cpus > 1 and len(msa_files) > 1:
109
+ # fork is used so workers share the MSA in memory
110
+ ctx = mp.get_context('fork')
111
+ with ctx.Pool(min(self.cpus, len(msa_files))) as pool:
112
+ for _ in tqdm(pool.imap_unordered(_write_replicate_task, msa_files),
113
+ total=len(msa_files), unit=' replicates'):
114
+ pass
115
+ else:
116
+ for msa_file in tqdm(msa_files, unit=' replicates'):
117
+ _write_replicate_task(msa_file)
118
+ finally:
119
+ _SHARED.clear()
120
+
121
+ def run(self, input_tree,
122
+ msa_file,
123
+ marker_info_file,
124
+ mask_file,
125
+ perc_markers_to_keep,
126
+ num_replicates,
127
+ model,
128
+ jk_dir,
129
+ output_dir):
130
+ """Jackknife marker genes.
131
+
132
+ Marker file should have the format:
133
+ <marker id>\t<marker name>\t<marker desc>\t<length>\n
134
+
135
+ Parameters
136
+ ----------
137
+ input_tree : str
138
+ Tree inferred with all data.
139
+ msa_file : str
140
+ File containing multiple sequence alignment for all taxa.
141
+ marker_info_file : str
142
+ File indicating database id, HMM name, description and length of each marker in the alignment.
143
+ mask_file : str
144
+ File indicating masking of multiple sequence alignment.
145
+ perc_markers_to_keep : float [0, 1]
146
+ Percentage of marker genes to keep in each replicate.
147
+ num_replicates : int
148
+ Number of replicates to perform.
149
+ model : str
150
+ Desired model of evolution.
151
+ jk_dir : str
152
+ Directory with pre-computed jackknife trees.
153
+ output_dir : str
154
+ Output directory for jackkife trees.
155
+ """
156
+
157
+ assert(model in ['wag', 'jtt'])
158
+
159
+ self.model = model
160
+ self.perc_markers_to_keep = perc_markers_to_keep
161
+
162
+ rep_tree_files = []
163
+ if not jk_dir:
164
+ self.replicate_dir = os.path.join(output_dir, 'replicates')
165
+ make_sure_path_exists(self.replicate_dir)
166
+
167
+ # determine length of each marker gene in alignment
168
+ marker_lengths = []
169
+ total_len = 0
170
+ with open(marker_info_file) as f:
171
+ f.readline()
172
+ for line in f:
173
+ line_split = line.split('\t')
174
+ ml = int(line_split[3])
175
+ marker_lengths.append(ml)
176
+ total_len += ml
177
+
178
+ self.logger.info('Concatenated length of markers: %d' % total_len)
179
+
180
+ # read mask
181
+ with open(mask_file) as f:
182
+ mask = f.readline().strip()
183
+ start = 0
184
+ self.marker_lengths = []
185
+ total_mask_len = 0
186
+ for ml in marker_lengths:
187
+ end = start + ml
188
+ zeros = mask[start:end].count('0')
189
+ start = end
190
+
191
+ self.marker_lengths.append(ml - zeros)
192
+ total_mask_len += ml - zeros
193
+
194
+ self.logger.info('Concatenated length of filtered MSA: %d' % total_mask_len)
195
+
196
+ # read full multiple sequence alignment
197
+ self.msa = seq_io.read(msa_file)
198
+
199
+ if len(list(self.msa.values())[0]) != total_mask_len:
200
+ self.logger.error('Length of MSA does not meet length of mask.')
201
+ sys.exit(1)
202
+
203
+ # create jackknifed alignments and infer trees
204
+ msa_files = [os.path.join(self.replicate_dir, f'jk_markers.msa.{i}.faa') for i in range(num_replicates)]
205
+ rep_tree_files = [os.path.join(self.replicate_dir, f'jk_markers.tree.{i}.tre') for i in range(num_replicates)]
206
+
207
+ self.logger.info('Calculating jackknife marker replicates:')
208
+ self._create_replicates(msa_files)
209
+ self.msa = None
210
+
211
+ self.logger.info(f'Inferring {num_replicates:,} jackknife trees using {self.cpus:,} CPUs:')
212
+ try:
213
+ infer_trees(msa_files, rep_tree_files, cpus=self.cpus, base_type='prot', model=self.model)
214
+ except FastTreeError as e:
215
+ self.logger.error(str(e))
216
+ sys.exit(1)
217
+ else:
218
+ for f in sorted(os.listdir(jk_dir)):
219
+ if f.endswith('.tree') or f.endswith('.tre'):
220
+ rep_tree_files.append(os.path.join(jk_dir, f))
221
+
222
+ self.logger.info('Read %d jackknife replicates.' % len(rep_tree_files))
223
+ if not rep_tree_files:
224
+ self.logger.error(f'No jackknife trees found in: {jk_dir}')
225
+ sys.exit(1)
226
+
227
+ # calculate support
228
+ self.logger.info(f'Calculating support from {len(rep_tree_files):,} replicates.')
229
+ output_tree = os.path.join(output_dir, Path(input_tree).stem + '.jk_markers.tree')
230
+ bootstrap_merge_replicates(input_tree, output_tree, rep_tree_files, self.cpus)
231
+
232
+ return output_tree
@@ -0,0 +1,136 @@
1
+ ###############################################################################
2
+ # #
3
+ # This program is free software: you can redistribute it and/or modify #
4
+ # it under the terms of the GNU General Public License as published by #
5
+ # the Free Software Foundation, either version 3 of the License, or #
6
+ # (at your option) any later version. #
7
+ # #
8
+ # This program is distributed in the hope that it will be useful, #
9
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of #
10
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
11
+ # GNU General Public License for more details. #
12
+ # #
13
+ # You should have received a copy of the GNU General Public License #
14
+ # along with this program. If not, see <http://www.gnu.org/licenses/>. #
15
+ # #
16
+ ###############################################################################
17
+
18
+ import os
19
+ import logging
20
+ import random
21
+ from math import floor
22
+ from pathlib import Path
23
+
24
+ from gtdblib.util.bio import seq_io
25
+ from gtdblib.util.shell.gtdbshutil import make_sure_path_exists
26
+
27
+ from genometreetk.fasttree import infer_trees
28
+
29
+ from genometreetk.tree_support import TreeSupport
30
+
31
+
32
+ class JackknifeTaxa(object):
33
+ """Assess robustness by jackknifing taxa in alignment."""
34
+
35
+ def __init__(self, cpus):
36
+ """Initialization.
37
+
38
+ Parameters
39
+ ----------
40
+ cpus : int
41
+ Number of cpus to use.
42
+ """
43
+
44
+ self.logger = logging.getLogger()
45
+
46
+ self.cpus = cpus
47
+
48
+ def _create_replicates(self, num_replicates):
49
+ """Create jackknifed alignments and infer a tree for each."""
50
+
51
+ msa_files = [os.path.join(self.replicate_dir, f'jk_taxa.msa.{i}.fna') for i in range(num_replicates)]
52
+ tree_files = [os.path.join(self.replicate_dir, f'jk_taxa.tree.{i}.tre') for i in range(num_replicates)]
53
+ for msa_file in msa_files:
54
+ self.jackknife_taxa(self.msa, self.perc_taxa_to_keep, self.outgroup_ids, msa_file)
55
+
56
+ infer_trees(msa_files, tree_files, cpus=self.cpus, base_type='prot', model=self.model)
57
+
58
+ def jackknife_taxa(self, msa, perc_taxa_to_keep, outgroup_ids, output_file):
59
+ """Jackknife alignment to a subset of taxa.
60
+
61
+ Parameters
62
+ ----------
63
+ msa : d[seq_id] -> seq
64
+ Full multiple sequence alignment.
65
+ perc_taxa_to_keep : float
66
+ Percentage of marker genes to keep in each replicate.
67
+ outgroup_ids : set
68
+ Labels of outgroup taxa.
69
+ output_file : str
70
+ File to write bootstrapped alignment.
71
+ """
72
+
73
+ # randomly select ingroup taxa
74
+ ingroup_taxa = set(msa.keys()) - outgroup_ids
75
+ taxa_to_keep = random.sample(sorted(ingroup_taxa), int(floor(len(ingroup_taxa) * perc_taxa_to_keep)))
76
+
77
+ taxa_to_keep = set(taxa_to_keep).union(outgroup_ids)
78
+
79
+ fout = open(output_file, 'w')
80
+ for seq_id, seq in msa.items():
81
+ if seq_id in taxa_to_keep:
82
+ fout.write('>' + seq_id + '\n')
83
+ fout.write(seq + '\n')
84
+ fout.close()
85
+
86
+ def run(self, input_tree, msa_file, outgroup_file, perc_taxa_to_keep, num_replicates, model, output_dir):
87
+ """Jackknife taxa.
88
+
89
+ Parameters
90
+ ----------
91
+ input_tree : str
92
+ Tree inferred with all data.
93
+ msa_file : str
94
+ File containing multiple sequence alignment for all taxa.
95
+ outgroup_file : str
96
+ File indicating labels of outgroup taxa.
97
+ perc_taxa_to_keep : float
98
+ Percentage of taxa to keep in each replicate.
99
+ num_replicates : int
100
+ Number of replicates to perform.
101
+ model : str
102
+ Desired model of evolution.
103
+ output_dir : str
104
+ input_tree directory for bootstrap trees.
105
+ """
106
+
107
+ assert(model in ['wag', 'jtt'])
108
+
109
+ self.perc_taxa_to_keep = perc_taxa_to_keep
110
+ self.model = model
111
+ self.replicate_dir = os.path.join(output_dir, 'replicates')
112
+ make_sure_path_exists(self.replicate_dir)
113
+
114
+ # read outgroup taxa
115
+ self.outgroup_ids = set()
116
+ if outgroup_file:
117
+ for line in open(outgroup_file):
118
+ self.outgroup_ids.add(line.strip())
119
+
120
+ # read full multiple sequence alignment
121
+ self.msa = seq_io.read(msa_file)
122
+
123
+ # calculate replicates
124
+ #***self.logger.info('Calculating jackknife taxa replicates:')
125
+ #***self._create_replicates(num_replicates)
126
+
127
+ # calculate support
128
+ rep_tree_files = []
129
+ for rep_index in range(num_replicates):
130
+ rep_tree_files.append(os.path.join(self.replicate_dir, 'jk_taxa.tree.' + str(rep_index) + '.tre'))
131
+
132
+ tree_support = TreeSupport()
133
+ output_tree = os.path.join(output_dir, Path(input_tree).stem + '.jk_taxa.tree')
134
+ tree_support.subset_taxa(input_tree, rep_tree_files, output_tree)
135
+
136
+ return output_tree