phyddle 0.0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phyddle/__init__.py +33 -0
- phyddle/__main__.py +8 -0
- phyddle/command_line.py +53 -0
- phyddle/estimate.py +409 -0
- phyddle/format.py +1052 -0
- phyddle/pipeline_step.py +52 -0
- phyddle/plot.py +1021 -0
- phyddle/simulate.py +242 -0
- phyddle/train.py +777 -0
- phyddle/utilities.py +2011 -0
- phyddle-0.0.7.dist-info/METADATA +132 -0
- phyddle-0.0.7.dist-info/RECORD +15 -0
- phyddle-0.0.7.dist-info/WHEEL +4 -0
- phyddle-0.0.7.dist-info/entry_points.txt +2 -0
- phyddle-0.0.7.dist-info/licenses/LICENSE +21 -0
phyddle/format.py
ADDED
|
@@ -0,0 +1,1052 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
"""
|
|
3
|
+
format
|
|
4
|
+
======
|
|
5
|
+
Defines classes and methods for the Format step. This step converts raw data
|
|
6
|
+
into tensor data that can be used by the Train step.
|
|
7
|
+
|
|
8
|
+
Authors: Michael Landis and Ammon Thompson
|
|
9
|
+
Copyright: (c) 2022-2023, Michael Landis and Ammon Thompson
|
|
10
|
+
License: MIT
|
|
11
|
+
"""
|
|
12
|
+
# standard imports
|
|
13
|
+
import copy
|
|
14
|
+
import os
|
|
15
|
+
import sys
|
|
16
|
+
|
|
17
|
+
# external imports
|
|
18
|
+
import dendropy as dp
|
|
19
|
+
import h5py
|
|
20
|
+
import numpy as np
|
|
21
|
+
import scipy as sp
|
|
22
|
+
import pandas as pd
|
|
23
|
+
from multiprocessing import Pool, set_start_method, cpu_count
|
|
24
|
+
from tqdm import tqdm
|
|
25
|
+
|
|
26
|
+
# phyddle imports
|
|
27
|
+
from phyddle import utilities as util
|
|
28
|
+
|
|
29
|
+
# Uncomment to debug multiprocessing
|
|
30
|
+
# import multiprocessing.util as mp_util
|
|
31
|
+
# mp_util.log_to_stderr(mp_util.SUBDEBUG)
|
|
32
|
+
|
|
33
|
+
# Allows multiprocessing fork (not spawn) new processes on Unix-based OS.
|
|
34
|
+
# However, import phyddle.simulate also calls set_start_method('fork'), and the
|
|
35
|
+
# function throws RuntimeError when called the 2nd+ time within a single Python.
|
|
36
|
+
# We handle this with a try-except block.
|
|
37
|
+
try:
|
|
38
|
+
set_start_method('fork')
|
|
39
|
+
except RuntimeError:
|
|
40
|
+
pass
|
|
41
|
+
|
|
42
|
+
#------------------------------------------------------------------------------#
|
|
43
|
+
|
|
44
|
+
def load(args):
|
|
45
|
+
"""
|
|
46
|
+
Load a Formatter object.
|
|
47
|
+
|
|
48
|
+
This function creates an instance of the Formatter class, initialized using
|
|
49
|
+
phyddle settings stored in args (dict).
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
args (dict): Contains phyddle settings.
|
|
53
|
+
"""
|
|
54
|
+
# settings
|
|
55
|
+
sys.setrecursionlimit(10000)
|
|
56
|
+
|
|
57
|
+
# load object
|
|
58
|
+
format_method = 'default'
|
|
59
|
+
if format_method == 'default':
|
|
60
|
+
return Formatter(args)
|
|
61
|
+
else:
|
|
62
|
+
return NotImplementedError
|
|
63
|
+
|
|
64
|
+
#------------------------------------------------------------------------------#
|
|
65
|
+
|
|
66
|
+
class Formatter:
|
|
67
|
+
"""
|
|
68
|
+
Class for formatting phylogenetic datasets and converting them into tensors
|
|
69
|
+
to be used by the Train step.
|
|
70
|
+
"""
|
|
71
|
+
def __init__(self, args): #, mdl):
|
|
72
|
+
"""
|
|
73
|
+
Initializes a new Formatter object.
|
|
74
|
+
|
|
75
|
+
Args:
|
|
76
|
+
args (dict): Contains phyddle settings.
|
|
77
|
+
"""
|
|
78
|
+
# initialize with phyddle settings
|
|
79
|
+
self.set_args(args)
|
|
80
|
+
# directory for simulations (input)
|
|
81
|
+
self.sim_proj_dir = f'{self.sim_dir}/{self.sim_proj}'
|
|
82
|
+
# directory for formatted tensors (output)
|
|
83
|
+
self.fmt_proj_dir = f'{self.fmt_dir}/{self.fmt_proj}'
|
|
84
|
+
# set number of processors
|
|
85
|
+
if self.num_proc <= 0:
|
|
86
|
+
self.num_proc = cpu_count() + self.num_proc
|
|
87
|
+
# run() attempts to generate one simulation per value in rep_idx,
|
|
88
|
+
# where rep_idx is list of unique ints to identify simulated datasets
|
|
89
|
+
if self.encode_all_sim:
|
|
90
|
+
self.rep_idx = self.get_rep_idx()
|
|
91
|
+
else:
|
|
92
|
+
self.rep_idx = list(range(self.start_idx, self.end_idx))
|
|
93
|
+
|
|
94
|
+
# split_idx is later assigned a value, after raw data is encoded
|
|
95
|
+
self.split_idx = {}
|
|
96
|
+
|
|
97
|
+
# get size of CPV+S tensors
|
|
98
|
+
self.num_tree_row = util.get_num_tree_row(self.tree_encode,
|
|
99
|
+
self.brlen_encode)
|
|
100
|
+
self.num_char_row = util.get_num_char_row(self.char_encode,
|
|
101
|
+
self.num_char,
|
|
102
|
+
self.num_states)
|
|
103
|
+
self.num_data_row = self.num_tree_row + self.num_char_row
|
|
104
|
+
|
|
105
|
+
# create logger to track runtime info
|
|
106
|
+
self.logger = util.Logger(args)
|
|
107
|
+
# done
|
|
108
|
+
return
|
|
109
|
+
|
|
110
|
+
def set_args(self, args):
|
|
111
|
+
"""
|
|
112
|
+
Assigns phyddle settings as Formatter attributes.
|
|
113
|
+
|
|
114
|
+
Args:
|
|
115
|
+
args (dict): Contains phyddle settings.
|
|
116
|
+
|
|
117
|
+
"""
|
|
118
|
+
# formatter arguments
|
|
119
|
+
self.args = args
|
|
120
|
+
step_args = util.make_step_args('F', args)
|
|
121
|
+
for k,v in step_args.items():
|
|
122
|
+
setattr(self, k, v)
|
|
123
|
+
|
|
124
|
+
# special case
|
|
125
|
+
self.tree_width_cats = [ self.tree_width ] # will be removed
|
|
126
|
+
|
|
127
|
+
return
|
|
128
|
+
|
|
129
|
+
def run(self):
|
|
130
|
+
"""
|
|
131
|
+
Formats all raw datasets into tensors.
|
|
132
|
+
|
|
133
|
+
This method prints status updates, creates the target directory for new
|
|
134
|
+
tensors, then formats each simulated raw dataset into an individual
|
|
135
|
+
tensor-formatted dataset, then combines all individual tensors into
|
|
136
|
+
a single tensor that contains all training examples.
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
verbose = self.verbose
|
|
140
|
+
|
|
141
|
+
# print header
|
|
142
|
+
util.print_step_header('fmt', [self.sim_proj_dir], self.fmt_proj_dir, verbose)
|
|
143
|
+
|
|
144
|
+
# prepare workspace
|
|
145
|
+
os.makedirs(self.fmt_proj_dir, exist_ok=True)
|
|
146
|
+
|
|
147
|
+
# start time
|
|
148
|
+
start_time,start_time_str = util.get_time()
|
|
149
|
+
util.print_str(f'▪ Start time of {start_time_str}', verbose)
|
|
150
|
+
|
|
151
|
+
# encode each dataset into individual tensors
|
|
152
|
+
util.print_str('▪ Encoding raw data as tensors', verbose)
|
|
153
|
+
self.encode_all()
|
|
154
|
+
|
|
155
|
+
# write tensors across all examples to file
|
|
156
|
+
util.print_str('▪ Combining and writing tensors', verbose)
|
|
157
|
+
self.write_tensor()
|
|
158
|
+
|
|
159
|
+
# end time
|
|
160
|
+
end_time,end_time_str = util.get_time()
|
|
161
|
+
run_time = util.get_time_diff(start_time, end_time)
|
|
162
|
+
# util.print_str(f'▪ End time: {end_time_str}', verbose)
|
|
163
|
+
util.print_str(f'▪ End time of {end_time_str} (+{run_time})', verbose)
|
|
164
|
+
|
|
165
|
+
# done
|
|
166
|
+
util.print_str('... done!', verbose)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def make_settings_str(self, idx, tree_width):
|
|
170
|
+
"""
|
|
171
|
+
Construct a string of settings for a single replicate.
|
|
172
|
+
|
|
173
|
+
Args:
|
|
174
|
+
idx (int): The replicate index.
|
|
175
|
+
tree_width (int): The tree width.
|
|
176
|
+
|
|
177
|
+
Returns:
|
|
178
|
+
str: The settings string.
|
|
179
|
+
"""
|
|
180
|
+
s = 'setting,value\n'
|
|
181
|
+
s += f'sim_proj_dir,{self.sim_proj_dir}\n'
|
|
182
|
+
s += f'fmt_proj_dir,{self.fmt_proj_dir}\n'
|
|
183
|
+
s += f'replicate_index,{idx}\n'
|
|
184
|
+
s += f'tree_width,{tree_width}\n'
|
|
185
|
+
return s
|
|
186
|
+
|
|
187
|
+
def encode_all(self):
|
|
188
|
+
"""
|
|
189
|
+
Encode each simulated replicate into its own matrix format.
|
|
190
|
+
|
|
191
|
+
Encode each simulated dataset identified by the replicate-index list
|
|
192
|
+
(self.rep_idx). Each dataset is encoded by calling self.encode_one(idx)
|
|
193
|
+
where idx is a unique value in self.rep_idx.
|
|
194
|
+
|
|
195
|
+
Encoded simulations are then sorted by tree-width category and stored
|
|
196
|
+
into the phy_tensors dictionary for later processing. Names and lengths
|
|
197
|
+
of label and summary statistic lists are also saved.
|
|
198
|
+
|
|
199
|
+
When self.use_parallel is True then all jobs are run in parallel via
|
|
200
|
+
multiprocessing.Pool. When self.use_parallel is false, jobs are run
|
|
201
|
+
serially with one CPU.
|
|
202
|
+
|
|
203
|
+
"""
|
|
204
|
+
|
|
205
|
+
# construct list of encoding arguments
|
|
206
|
+
args = []
|
|
207
|
+
|
|
208
|
+
for idx in self.rep_idx:
|
|
209
|
+
args.append((f'{self.sim_proj_dir}/sim.{idx}', idx))
|
|
210
|
+
|
|
211
|
+
# visit each replicate, encode it, and return result
|
|
212
|
+
if self.use_parallel:
|
|
213
|
+
# parallel jobs
|
|
214
|
+
with Pool(processes=self.num_proc) as pool:
|
|
215
|
+
# Note, it's critical to call this as list(tqdm(pool.imap(...)))
|
|
216
|
+
# - pool.imap runs the parallelization
|
|
217
|
+
# - tqdm generates progress bar
|
|
218
|
+
# - list acts as a finalizer for the pool.imap work
|
|
219
|
+
# Have not benchmarked imap/imap_unordered, chunksize, etc.
|
|
220
|
+
res = list(tqdm(pool.imap(self.encode_one_star,
|
|
221
|
+
args, chunksize=5),
|
|
222
|
+
total=len(args),
|
|
223
|
+
desc='Encoding',
|
|
224
|
+
smoothing=0))
|
|
225
|
+
|
|
226
|
+
else:
|
|
227
|
+
# serial jobs
|
|
228
|
+
res = [ self.encode_one_star(a) for a in tqdm(args,
|
|
229
|
+
total=len(args),
|
|
230
|
+
desc='Encoding',
|
|
231
|
+
smoothing=0) ]
|
|
232
|
+
|
|
233
|
+
# save all phylogenetic-state tensors into the phy_tensors dictionary,
|
|
234
|
+
# while sorting tensors into different tree-width categories
|
|
235
|
+
self.phy_tensors = {}
|
|
236
|
+
# for size in self.tree_width_cats:
|
|
237
|
+
# self.phy_tensors[size] = {}
|
|
238
|
+
for i in range(len(res)):
|
|
239
|
+
if res[i] is not None:
|
|
240
|
+
# tensor_size = res[i].shape[1]
|
|
241
|
+
# self.phy_tensors[tensor_size][i] = res[i]
|
|
242
|
+
self.phy_tensors[i] = res[i]
|
|
243
|
+
|
|
244
|
+
# save names/lengths of summary statistic and label lists
|
|
245
|
+
self.summ_stat_names = self.get_summ_stat_names()
|
|
246
|
+
self.label_names = self.get_label_names()
|
|
247
|
+
self.num_summ_stat = len(self.summ_stat_names)
|
|
248
|
+
self.num_labels = len(self.label_names)
|
|
249
|
+
|
|
250
|
+
return
|
|
251
|
+
|
|
252
|
+
def get_rep_idx(self):
|
|
253
|
+
rep_idx = set()
|
|
254
|
+
files = os.listdir(f'{self.sim_proj_dir}')
|
|
255
|
+
for f in files:
|
|
256
|
+
rep_idx.add(int(f.split('.')[1]))
|
|
257
|
+
rep_idx = sorted(list(rep_idx))
|
|
258
|
+
return rep_idx
|
|
259
|
+
|
|
260
|
+
def get_summ_stat_names(self):
|
|
261
|
+
"""
|
|
262
|
+
Get names of summary statistics from first representative file.
|
|
263
|
+
|
|
264
|
+
Returns:
|
|
265
|
+
ret (list): List of summary statistics names.
|
|
266
|
+
"""
|
|
267
|
+
# get first representative file
|
|
268
|
+
idx = None
|
|
269
|
+
# for i in self.tree_width_cats:
|
|
270
|
+
# k_list = list(self.phy_tensors[i].keys())
|
|
271
|
+
# if len(k_list) > 0 and idx is None:
|
|
272
|
+
# idx = k_list[0]
|
|
273
|
+
k_list = list(self.phy_tensors.keys())
|
|
274
|
+
if len(k_list) > 0 and idx is None:
|
|
275
|
+
idx = k_list[0]
|
|
276
|
+
|
|
277
|
+
fn = f'{self.sim_proj_dir}/sim.{idx}.summ_stat.csv'
|
|
278
|
+
df = pd.read_csv(fn,header=0)
|
|
279
|
+
ret = df.columns.to_list()
|
|
280
|
+
return ret
|
|
281
|
+
|
|
282
|
+
def get_label_names(self):
|
|
283
|
+
"""
|
|
284
|
+
Get names of training labels from first representative file.
|
|
285
|
+
|
|
286
|
+
Returns:
|
|
287
|
+
ret (list): List of label names.
|
|
288
|
+
"""
|
|
289
|
+
# get first representative file
|
|
290
|
+
idx = None
|
|
291
|
+
# for i in self.tree_width_cats:
|
|
292
|
+
# k_list = list(self.phy_tensors[i].keys())
|
|
293
|
+
# if len(k_list) > 0 and idx is None:
|
|
294
|
+
# idx = k_list[0]
|
|
295
|
+
k_list = list(self.phy_tensors.keys())
|
|
296
|
+
if len(k_list) > 0 and idx is None:
|
|
297
|
+
idx = k_list[0]
|
|
298
|
+
|
|
299
|
+
fn = f'{self.sim_proj_dir}/sim.{idx}.param_row.csv'
|
|
300
|
+
df = pd.read_csv(fn,header=0)
|
|
301
|
+
ret = df.columns.to_list()
|
|
302
|
+
return ret
|
|
303
|
+
|
|
304
|
+
def split_examples(self):
|
|
305
|
+
"""
|
|
306
|
+
Split examples into training and test datasets.
|
|
307
|
+
"""
|
|
308
|
+
split_idx = {}
|
|
309
|
+
# tree_width = self.tree_width_cats[0]
|
|
310
|
+
# rep_idx = sorted(list(self.phy_tensors[tree_width]))
|
|
311
|
+
rep_idx = sorted(list(self.phy_tensors.keys()))
|
|
312
|
+
num_samples = len(rep_idx)
|
|
313
|
+
np.random.shuffle(rep_idx)
|
|
314
|
+
num_test = int(num_samples * self.prop_test)
|
|
315
|
+
split_idx['test'] = rep_idx[:num_test]
|
|
316
|
+
split_idx['train'] = rep_idx[num_test:]
|
|
317
|
+
|
|
318
|
+
return split_idx
|
|
319
|
+
|
|
320
|
+
def write_tensor(self):
|
|
321
|
+
"""
|
|
322
|
+
Write the tensor in csv or hdf5 format.
|
|
323
|
+
"""
|
|
324
|
+
self.split_idx = self.split_examples()
|
|
325
|
+
if self.tensor_format == 'csv':
|
|
326
|
+
# self.write_tensor_csv()
|
|
327
|
+
self.write_tensor_csv('train')
|
|
328
|
+
self.write_tensor_csv('test')
|
|
329
|
+
elif self.tensor_format == 'hdf5':
|
|
330
|
+
# self.write_tensor_hdf5()
|
|
331
|
+
self.write_tensor_hdf5('train')
|
|
332
|
+
self.write_tensor_hdf5('test')
|
|
333
|
+
return
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def write_tensor_hdf5(self, data_str):
|
|
337
|
+
"""
|
|
338
|
+
Writes data to HDF5 file for each tree width.
|
|
339
|
+
|
|
340
|
+
This class creates HDF5 files
|
|
341
|
+
|
|
342
|
+
"""
|
|
343
|
+
|
|
344
|
+
assert(data_str in ['test', 'train'])
|
|
345
|
+
|
|
346
|
+
# build files
|
|
347
|
+
tree_width = self.tree_width #self.tree_width_cats[0]
|
|
348
|
+
#phy_tensor = self.phy_tensors[tree_width]
|
|
349
|
+
|
|
350
|
+
# dimensions
|
|
351
|
+
rep_idx = self.split_idx[data_str]
|
|
352
|
+
num_samples = len(rep_idx)
|
|
353
|
+
num_data_length = tree_width * self.num_data_row
|
|
354
|
+
|
|
355
|
+
# print info
|
|
356
|
+
print(f'Making {data_str} hdf5 dataset: {num_samples} examples for tree width = {tree_width}')
|
|
357
|
+
|
|
358
|
+
# HDF5 file
|
|
359
|
+
out_hdf5_fn = f'{self.fmt_proj_dir}/{data_str}.nt{tree_width}.hdf5'
|
|
360
|
+
hdf5_file = h5py.File(out_hdf5_fn, 'w')
|
|
361
|
+
|
|
362
|
+
# create datasets for numerical data
|
|
363
|
+
dat_data = hdf5_file.create_dataset('phy_data',
|
|
364
|
+
(num_samples, num_data_length),
|
|
365
|
+
dtype='f', compression='gzip')
|
|
366
|
+
dat_stat = hdf5_file.create_dataset('summ_stat',
|
|
367
|
+
(num_samples, self.num_summ_stat),
|
|
368
|
+
dtype='f', compression='gzip')
|
|
369
|
+
dat_labels = hdf5_file.create_dataset('labels',
|
|
370
|
+
(num_samples, self.num_labels),
|
|
371
|
+
dtype='f', compression='gzip')
|
|
372
|
+
|
|
373
|
+
# the replicates for this tree width
|
|
374
|
+
#_rep_idx = list(phy_tensor.keys())
|
|
375
|
+
|
|
376
|
+
# load all the info
|
|
377
|
+
res = [ self.load_one_sim(idx=idx, tree_width=tree_width) for idx in tqdm(rep_idx,
|
|
378
|
+
total=len(rep_idx),
|
|
379
|
+
desc='Combining',
|
|
380
|
+
smoothing=0) ]
|
|
381
|
+
|
|
382
|
+
# store all numerical data into hdf5)
|
|
383
|
+
if len(res) > 0:
|
|
384
|
+
dat_data[:,:] = np.vstack( [ x[0] for x in res ] )
|
|
385
|
+
dat_stat[:,:] = np.vstack( [ x[1] for x in res ] )
|
|
386
|
+
dat_labels[:,:] = np.vstack( [ x[2] for x in res ] )
|
|
387
|
+
|
|
388
|
+
# read in summ_stats and labels (_all_ params) dataframes
|
|
389
|
+
df_summ_stats = pd.DataFrame(dat_stat, columns=self.summ_stat_names)
|
|
390
|
+
df_labels = pd.DataFrame(dat_labels, columns=self.label_names)
|
|
391
|
+
|
|
392
|
+
# separate data parameters (things we know) from label parameters (things we estimate)
|
|
393
|
+
df_labels_new = df_labels[self.param_est]
|
|
394
|
+
df_labels_move = df_labels[self.param_data]
|
|
395
|
+
|
|
396
|
+
# concatenate new data parameters as column to existing summ_stats dataframe
|
|
397
|
+
df_aux_data = df_summ_stats.join( df_labels_move )
|
|
398
|
+
|
|
399
|
+
# get new label/stat names
|
|
400
|
+
new_label_names = self.param_est
|
|
401
|
+
new_aux_data_names = self.summ_stat_names + self.param_data
|
|
402
|
+
|
|
403
|
+
# delete original datasets
|
|
404
|
+
del hdf5_file['summ_stat']
|
|
405
|
+
del hdf5_file['labels']
|
|
406
|
+
|
|
407
|
+
# create new datasets
|
|
408
|
+
hdf5_file.create_dataset('labels', df_labels_new.shape, 'f', df_labels_new, compression='gzip')
|
|
409
|
+
hdf5_file.create_dataset('label_names', (1, len(new_label_names)), 'S64', new_label_names, compression='gzip')
|
|
410
|
+
hdf5_file.create_dataset('aux_data', df_aux_data.shape, 'f', df_aux_data, compression='gzip')
|
|
411
|
+
hdf5_file.create_dataset('aux_data_names', (1, len(new_aux_data_names)), 'S64', new_aux_data_names, compression='gzip')
|
|
412
|
+
|
|
413
|
+
# close HDF5 files
|
|
414
|
+
hdf5_file.close()
|
|
415
|
+
|
|
416
|
+
return
|
|
417
|
+
|
|
418
|
+
def write_tensor_csv(self, data_str):
|
|
419
|
+
"""
|
|
420
|
+
Writes CSV files for phylogenetic tensors.
|
|
421
|
+
|
|
422
|
+
The method iterates through the phylogenetic tensors for each tree width
|
|
423
|
+
and generates CSV files containing the tensor data and labels.
|
|
424
|
+
"""
|
|
425
|
+
assert(data_str in ['test', 'train'])
|
|
426
|
+
|
|
427
|
+
# build files
|
|
428
|
+
tree_width = self.tree_width #_cats[0]
|
|
429
|
+
phy_tensor = self.phy_tensors #[tree_width]
|
|
430
|
+
|
|
431
|
+
# dimensions
|
|
432
|
+
rep_idx = self.split_idx[data_str]
|
|
433
|
+
num_samples = len(rep_idx)
|
|
434
|
+
|
|
435
|
+
# info
|
|
436
|
+
print(f'Making {data_str} csv dataset: {num_samples} examples for tree width = {tree_width}')
|
|
437
|
+
|
|
438
|
+
# output csv filepaths
|
|
439
|
+
#out_hdf5_fn = f'{self.fmt_proj_dir}/{data_str}.nt{tree_width}.hdf5'
|
|
440
|
+
out_prefix = f'{self.fmt_proj_dir}/{data_str}.nt{tree_width}'
|
|
441
|
+
in_prefix = f'{self.sim_proj_dir}/sim'
|
|
442
|
+
out_phys_fn = f'{out_prefix}.phy_data.csv'
|
|
443
|
+
out_stat_fn = f'{out_prefix}.aux_data.csv'
|
|
444
|
+
out_labels_fn = f'{out_prefix}.labels.csv'
|
|
445
|
+
|
|
446
|
+
# phylogenetic state tensor
|
|
447
|
+
with open(out_phys_fn, 'w') as outfile:
|
|
448
|
+
for idx in rep_idx:
|
|
449
|
+
pt = phy_tensor[idx]
|
|
450
|
+
#for j,(idx,pt) in enumerate(phy_tensor.items()):
|
|
451
|
+
s = ','.join(map(str, pt.flatten())) + '\n'
|
|
452
|
+
outfile.write(s)
|
|
453
|
+
|
|
454
|
+
# summary stats tensor
|
|
455
|
+
with open(out_stat_fn, 'w') as outfile:
|
|
456
|
+
is_first = True
|
|
457
|
+
for idx in rep_idx:
|
|
458
|
+
# for j,idx in enumerate(phy_tensor.keys()):
|
|
459
|
+
# if idx in rep_idx:
|
|
460
|
+
fname = f'{in_prefix}.{idx}.summ_stat.csv'
|
|
461
|
+
with open(fname, 'r') as infile:
|
|
462
|
+
if is_first:
|
|
463
|
+
s = infile.read()
|
|
464
|
+
is_first = False
|
|
465
|
+
else:
|
|
466
|
+
s = ''.join(infile.readlines()[1:])
|
|
467
|
+
outfile.write(s)
|
|
468
|
+
|
|
469
|
+
# labels input tensor
|
|
470
|
+
with open(out_labels_fn, 'w') as outfile:
|
|
471
|
+
is_first = True
|
|
472
|
+
for idx in rep_idx:
|
|
473
|
+
# for j,idx in enumerate(phy_tensor.keys()):
|
|
474
|
+
# if idx in rep_idx:
|
|
475
|
+
fname = f'{in_prefix}.{idx}.param_row.csv'
|
|
476
|
+
with open(fname, 'r') as infile:
|
|
477
|
+
if is_first:
|
|
478
|
+
s = infile.read()
|
|
479
|
+
is_first = False
|
|
480
|
+
else:
|
|
481
|
+
s = ''.join(infile.readlines()[1:])
|
|
482
|
+
outfile.write(s)
|
|
483
|
+
|
|
484
|
+
# rearrange labels and summary statistics
|
|
485
|
+
# - labels contains param_est
|
|
486
|
+
# - aux_data contains summ_stat and param_data
|
|
487
|
+
|
|
488
|
+
# read in summ_stats and labels
|
|
489
|
+
df_summ_stats = pd.read_csv(out_stat_fn)
|
|
490
|
+
df_labels = pd.read_csv(out_labels_fn)
|
|
491
|
+
|
|
492
|
+
# separate data parameters (things we know) from label parameters (things we predict)
|
|
493
|
+
df_labels_keep = df_labels[self.param_est]
|
|
494
|
+
df_labels_move = df_labels[self.param_data]
|
|
495
|
+
|
|
496
|
+
# concatenate new data parameters as column to existing summ_stats dataframe
|
|
497
|
+
df_summ_stats = df_summ_stats.join( df_labels_move )
|
|
498
|
+
|
|
499
|
+
# overwrite original files with new modified versions
|
|
500
|
+
df_summ_stats.to_csv(out_stat_fn, index=False)
|
|
501
|
+
df_labels_keep.to_csv(out_labels_fn, index=False)
|
|
502
|
+
|
|
503
|
+
return
|
|
504
|
+
|
|
505
|
+
def load_one_sim(self, idx, tree_width):
|
|
506
|
+
"""Load data for one simulation given its index and tree width.
|
|
507
|
+
|
|
508
|
+
Args:
|
|
509
|
+
idx (int): Index of the simulation.
|
|
510
|
+
tree_width (int): Tree width for the simulation.
|
|
511
|
+
|
|
512
|
+
Returns:
|
|
513
|
+
Tuple of numpy arrays (x1, x2, x3).
|
|
514
|
+
"""
|
|
515
|
+
fname_base = f'{self.sim_proj_dir}/sim.{idx}'
|
|
516
|
+
fname_param = fname_base + '.param_row.csv'
|
|
517
|
+
fname_stat = fname_base + '.summ_stat.csv'
|
|
518
|
+
# x1 = self.phy_tensors[tree_width][idx].flatten()
|
|
519
|
+
x1 = self.phy_tensors[idx].flatten()
|
|
520
|
+
x2 = np.loadtxt(fname_stat, delimiter=',', skiprows=1)
|
|
521
|
+
x3 = np.loadtxt(fname_param, delimiter=',', skiprows=1)
|
|
522
|
+
return (x1,x2,x3)
|
|
523
|
+
|
|
524
|
+
def encode_one_star(self, args):
|
|
525
|
+
"""Wrapper for encode_one w/ unpacked args"""
|
|
526
|
+
return self.encode_one(*args)
|
|
527
|
+
|
|
528
|
+
def encode_one(self, tmp_fn, idx, save_phyenc_csv=False):
|
|
529
|
+
"""
|
|
530
|
+
Encode a single simulated raw dataset into tensor format.
|
|
531
|
+
|
|
532
|
+
This function transforms raw input into tensor outputs. The inputs are
|
|
533
|
+
read from a tree file and data matrix file. The tree is filtered to
|
|
534
|
+
target ranges of taxon counts and pruned of non-extant taxa.
|
|
535
|
+
|
|
536
|
+
Trees are then binned into tree-width categories (number of columns for
|
|
537
|
+
compact vector representation). Next, trees and character data are
|
|
538
|
+
encoded into extended CBLV or CDV formats that contain additional state
|
|
539
|
+
and branch length information.
|
|
540
|
+
|
|
541
|
+
Arguments:
|
|
542
|
+
tmp_fn (str): prefix for individual dataset
|
|
543
|
+
idx (int): replicate index for simulation
|
|
544
|
+
save_phyenc_csv (bool): save phylogenetic state tensor as csv?
|
|
545
|
+
|
|
546
|
+
Returns:
|
|
547
|
+
cpvs (numpy.array): compact phylo vector + states (CPVS)
|
|
548
|
+
"""
|
|
549
|
+
NUM_DIGITS = 10
|
|
550
|
+
np.set_printoptions(formatter={'float': lambda x: format(x, '8.6E')},
|
|
551
|
+
precision=NUM_DIGITS)
|
|
552
|
+
|
|
553
|
+
# make filenames
|
|
554
|
+
dat_nex_fn = tmp_fn + '.dat.nex'
|
|
555
|
+
tre_fn = tmp_fn + '.tre'
|
|
556
|
+
prune_fn = tmp_fn + '.extant.tre'
|
|
557
|
+
down_fn = tmp_fn + '.downsampled.tre'
|
|
558
|
+
cpsv_fn = tmp_fn + '.phy_data.csv'
|
|
559
|
+
ss_fn = tmp_fn + '.summ_stat.csv'
|
|
560
|
+
info_fn = tmp_fn + '.info.csv'
|
|
561
|
+
|
|
562
|
+
# check if key files exist
|
|
563
|
+
err_msg = None
|
|
564
|
+
if not os.path.exists(dat_nex_fn):
|
|
565
|
+
err_msg = f'Formatter.encode_one(): {dat_nex_fn} does not exist'
|
|
566
|
+
print(err_msg)
|
|
567
|
+
self.logger.write_log('fmt', err_msg)
|
|
568
|
+
if not os.path.exists(tre_fn):
|
|
569
|
+
err_msg = f'Formatter.encode_one(): {tre_fn} does not exist'
|
|
570
|
+
print(err_msg)
|
|
571
|
+
self.logger.write_log('fmt', err_msg)
|
|
572
|
+
if err_msg is not None:
|
|
573
|
+
return
|
|
574
|
+
|
|
575
|
+
# read in nexus data file as numpy array
|
|
576
|
+
if self.char_format == 'nexus':
|
|
577
|
+
dat = util.convert_nexus_to_array(dat_nex_fn,
|
|
578
|
+
self.char_encode,
|
|
579
|
+
self.num_states)
|
|
580
|
+
elif self.char_format == 'csv':
|
|
581
|
+
dat = util.convert_csv_to_array(dat_nex_fn,
|
|
582
|
+
self.char_encode,
|
|
583
|
+
self.num_states)
|
|
584
|
+
|
|
585
|
+
# get tree file
|
|
586
|
+
phy = util.read_tree(tre_fn)
|
|
587
|
+
if phy is None:
|
|
588
|
+
return
|
|
589
|
+
|
|
590
|
+
# prune tree, if needed
|
|
591
|
+
if self.tree_encode == 'extant':
|
|
592
|
+
phy_prune = util.make_prune_phy(phy, prune_fn)
|
|
593
|
+
if phy_prune is None:
|
|
594
|
+
# abort, no valid pruned tree
|
|
595
|
+
return
|
|
596
|
+
else:
|
|
597
|
+
# valid pruned tree
|
|
598
|
+
phy = copy.deepcopy(phy_prune)
|
|
599
|
+
|
|
600
|
+
# downsample taxa
|
|
601
|
+
num_taxa_orig = len(phy.leaf_nodes())
|
|
602
|
+
phy = util.make_downsample_phy(
|
|
603
|
+
phy, down_fn,
|
|
604
|
+
max_taxa=max(self.tree_width_cats),
|
|
605
|
+
strategy=self.downsample_taxa)
|
|
606
|
+
|
|
607
|
+
# get tree size
|
|
608
|
+
num_taxa = len(phy.leaf_nodes())
|
|
609
|
+
if num_taxa > np.max(self.tree_width_cats):
|
|
610
|
+
# abort, too many taxa
|
|
611
|
+
return
|
|
612
|
+
if num_taxa < self.min_num_taxa or num_taxa < 0:
|
|
613
|
+
# abort, too few taxa
|
|
614
|
+
return
|
|
615
|
+
|
|
616
|
+
# get tree width from resulting vector
|
|
617
|
+
tree_width = util.find_tree_width(num_taxa, self.tree_width_cats)
|
|
618
|
+
|
|
619
|
+
# create compact phylo-state vector, CPV+S = {CBLV+S, CDV+S}
|
|
620
|
+
cpvs_data = None
|
|
621
|
+
|
|
622
|
+
# encode CBLV+S
|
|
623
|
+
cpvs_data = self.encode_cpvs(phy, dat, tree_width=tree_width,
|
|
624
|
+
tree_encode_type=self.brlen_encode,
|
|
625
|
+
tree_type=self.tree_encode)
|
|
626
|
+
|
|
627
|
+
# save CPVS
|
|
628
|
+
save_phyenc_csv_ = self.save_phyenc_csv or save_phyenc_csv
|
|
629
|
+
if save_phyenc_csv_ and cpvs_data is not None:
|
|
630
|
+
cpsv_str = util.make_clean_phyloenc_str(cpvs_data.flatten())
|
|
631
|
+
util.write_to_file(cpsv_str, cpsv_fn)
|
|
632
|
+
|
|
633
|
+
# record info
|
|
634
|
+
info_str = self.make_settings_str(idx, tree_width)
|
|
635
|
+
util.write_to_file(info_str, info_fn)
|
|
636
|
+
|
|
637
|
+
# record summ stat data
|
|
638
|
+
ss = self.make_summ_stat(phy, dat)
|
|
639
|
+
# add downsampling info
|
|
640
|
+
ss['num_taxa'] = num_taxa_orig
|
|
641
|
+
ss['prop_taxa'] = num_taxa / num_taxa_orig
|
|
642
|
+
|
|
643
|
+
# save summ. stats.
|
|
644
|
+
ss_str = self.make_summ_stat_str(ss)
|
|
645
|
+
util.write_to_file(ss_str, ss_fn)
|
|
646
|
+
|
|
647
|
+
# done!
|
|
648
|
+
return cpvs_data
|
|
649
|
+
|
|
650
|
+
def make_summ_stat(self, phy, dat):
|
|
651
|
+
"""
|
|
652
|
+
Generate summary statistics.
|
|
653
|
+
|
|
654
|
+
This function populates a dictionary of summary statistics. Keys
|
|
655
|
+
are the names of statistics that later appear as column names in the
|
|
656
|
+
summ_stat and aux_data files and containers. Values are for the
|
|
657
|
+
inputted phy and dat variables.
|
|
658
|
+
|
|
659
|
+
Arguments:
|
|
660
|
+
phy (dendropy.Tree): phylogenetic tree
|
|
661
|
+
dat (numpy.array): character data
|
|
662
|
+
|
|
663
|
+
Returns:
|
|
664
|
+
summ_stats (dict): summary statistics
|
|
665
|
+
"""
|
|
666
|
+
# new dictionary to return
|
|
667
|
+
summ_stats = {}
|
|
668
|
+
|
|
669
|
+
# read basic info from phylogenetic tree
|
|
670
|
+
num_taxa = len(phy.leaf_nodes())
|
|
671
|
+
node_ages = phy.internal_node_ages(ultrametricity_precision=False)
|
|
672
|
+
root_age = phy.seed_node.age
|
|
673
|
+
branch_lengths = [ nd.edge.length for nd in phy.nodes() if nd != phy.seed_node ]
|
|
674
|
+
#root_distances = phy.calc_node_root_distances()
|
|
675
|
+
#root_distances = [ nd.root_distance for nd in phy.nodes() if nd.is_leaf]
|
|
676
|
+
#phy.calc_node_ages(ultrametricity_precision=False)
|
|
677
|
+
#tree_height = np.max( root_distances )
|
|
678
|
+
|
|
679
|
+
# tree statistics
|
|
680
|
+
summ_stats['tree_length'] = phy.length()
|
|
681
|
+
summ_stats['root_age'] = root_age
|
|
682
|
+
summ_stats['brlen_mean'] = np.mean(branch_lengths)
|
|
683
|
+
summ_stats['brlen_var'] = np.var(branch_lengths)
|
|
684
|
+
summ_stats['brlen_skew'] = sp.stats.skew(branch_lengths)
|
|
685
|
+
summ_stats['age_mean'] = np.mean(node_ages)
|
|
686
|
+
summ_stats['age_var'] = np.var(node_ages)
|
|
687
|
+
summ_stats['age_skew'] = sp.stats.skew(node_ages)
|
|
688
|
+
summ_stats['B1'] = dp.calculate.treemeasure.B1(phy)
|
|
689
|
+
summ_stats['N_bar'] = dp.calculate.treemeasure.N_bar(phy)
|
|
690
|
+
summ_stats['colless'] = dp.calculate.treemeasure.colless_tree_imbalance(phy)
|
|
691
|
+
summ_stats['treeness'] = dp.calculate.treemeasure.treeness(phy)
|
|
692
|
+
|
|
693
|
+
# possible tree statistics, but not computable for arbitrary trees
|
|
694
|
+
#summ_stats['gamma'] = dp.calculate.treemeasure.pybus_harvey_gamma(phy)
|
|
695
|
+
#summ_stats['brlen_kurt'] = sp.stats.kurtosis(branch_lengths)
|
|
696
|
+
#summ_stats['age_kurt'] = sp.stats.kurtosis(root_distances)
|
|
697
|
+
#summ_stats['sackin'] = dp.calculate.treemeasure.sackin_index(phy)
|
|
698
|
+
|
|
699
|
+
# frequencies of character states
|
|
700
|
+
if self.char_encode == 'integer':
|
|
701
|
+
# integer-encoded states
|
|
702
|
+
for i in range(self.num_states):
|
|
703
|
+
summ_stats['f_dat_' + str(i)] = 0
|
|
704
|
+
unique, counts = np.unique(dat, return_counts=True)
|
|
705
|
+
for i,j in zip(unique, counts):
|
|
706
|
+
summ_stats['f_dat_' + str(i)] = j / num_taxa
|
|
707
|
+
|
|
708
|
+
elif self.char_encode == 'one_hot':
|
|
709
|
+
# one-hot-encoded states
|
|
710
|
+
for i in range(dat.shape[0]):
|
|
711
|
+
summ_stats['f_dat_' + str(i)] = np.sum(dat.iloc[i]) / num_taxa
|
|
712
|
+
|
|
713
|
+
# done
|
|
714
|
+
return summ_stats
|
|
715
|
+
|
|
716
|
+
# ==> Can probably move to util? seems generic and useful
|
|
717
|
+
def make_summ_stat_str(self, ss):
|
|
718
|
+
"""
|
|
719
|
+
Generate a string representation of the summary statistics.
|
|
720
|
+
|
|
721
|
+
Arguments:
|
|
722
|
+
ss (dict): dictionary of summary statistics
|
|
723
|
+
|
|
724
|
+
Returns:
|
|
725
|
+
keys_str: string containing the keys of the summary statistics
|
|
726
|
+
vals_str: string containing the values of the summary statistics
|
|
727
|
+
|
|
728
|
+
"""
|
|
729
|
+
keys_str = ','.join( list(ss.keys()) ) + '\n'
|
|
730
|
+
vals_str = ','.join( [ str(x) for x in ss.values() ] ) + '\n'
|
|
731
|
+
return keys_str + vals_str
|
|
732
|
+
|
|
733
|
+
def encode_cpvs(self, phy, dat, tree_width, tree_type, tree_encode_type, rescale=True):
|
|
734
|
+
"""
|
|
735
|
+
Encode Compact Phylogenetic Vector + States (CPV+S) array
|
|
736
|
+
|
|
737
|
+
This function encodes the dataset into Compact Bijective Ladderized
|
|
738
|
+
Vector + States (CBLV+S) when tree_type is 'serial' or Compact
|
|
739
|
+
Diversity-Reordered Vector + States (CDV+S) when tree_type is 'extant'.
|
|
740
|
+
|
|
741
|
+
Arguments:
|
|
742
|
+
phy (dendropy.Tree): phylogenetic tree
|
|
743
|
+
dat (numpy.array): character data
|
|
744
|
+
tree_width (int): number of columns (max. num. taxa) in CPVS array
|
|
745
|
+
tree_type (str): type of the tree ('serial' or 'extant')
|
|
746
|
+
tree_encode_type (str): type of tree encoding ('height_only' or 'height_brlen')
|
|
747
|
+
rescale: set tree height to 1 before encoding when True
|
|
748
|
+
|
|
749
|
+
Returns:
|
|
750
|
+
cpvs (numpy.array): CPV+S encoded tensor
|
|
751
|
+
"""
|
|
752
|
+
if tree_type == 'serial':
|
|
753
|
+
cpvs = self.encode_cblvs(phy, dat, tree_width, tree_encode_type, rescale)
|
|
754
|
+
elif tree_type == 'extant':
|
|
755
|
+
cpvs = self.encode_cdvs(phy, dat, tree_width, tree_encode_type, rescale)
|
|
756
|
+
else:
|
|
757
|
+
ValueError(f'Unrecognized {tree_type}')
|
|
758
|
+
return cpvs
|
|
759
|
+
|
|
760
|
+
def encode_cdvs(self, phy, dat, tree_width, tree_encode_type, rescale=True):
|
|
761
|
+
"""
|
|
762
|
+
Encode Compact Diversity-reordered Vector + States (CDV+S) array
|
|
763
|
+
|
|
764
|
+
# num columns equals tree_width, 0-padding
|
|
765
|
+
# returns tensor with following rows
|
|
766
|
+
# 0: internal node root-distance
|
|
767
|
+
# 1: leaf node branch length
|
|
768
|
+
# 2: internal node branch ength
|
|
769
|
+
# 3+: state encoding
|
|
770
|
+
|
|
771
|
+
Arguments:
|
|
772
|
+
phy (dendropy.Tree): phylogenetic tree
|
|
773
|
+
dat (numpy.array): character data
|
|
774
|
+
tree_width (int): number of columns (max. num. taxa) in CPVS array
|
|
775
|
+
tree_encode_type (str): type of tree encoding ('height_only' or 'height_brlen')
|
|
776
|
+
rescale: set tree height to 1 before encoding when True
|
|
777
|
+
|
|
778
|
+
Returns:
|
|
779
|
+
numpy.ndarray: The encoded CDVs tensor.
|
|
780
|
+
"""
|
|
781
|
+
|
|
782
|
+
# data dimensions
|
|
783
|
+
num_char_row = dat.shape[0]
|
|
784
|
+
if tree_encode_type == 'height_only':
|
|
785
|
+
num_tree_row = 1
|
|
786
|
+
elif tree_encode_type == 'height_brlen':
|
|
787
|
+
num_tree_row = 3
|
|
788
|
+
|
|
789
|
+
# initialize workspace
|
|
790
|
+
root_distances = phy.calc_node_root_distances(return_leaf_distances_only=False)
|
|
791
|
+
heights = np.zeros( (num_tree_row, tree_width) )
|
|
792
|
+
states = np.zeros( (num_char_row, tree_width) )
|
|
793
|
+
state_idx = 0
|
|
794
|
+
height_idx = 0
|
|
795
|
+
|
|
796
|
+
# postorder traversal to rotate nodes by clade-length
|
|
797
|
+
for nd in phy.postorder_node_iter():
|
|
798
|
+
if nd.is_leaf():
|
|
799
|
+
nd.treelen = 0.
|
|
800
|
+
else:
|
|
801
|
+
children = nd.child_nodes()
|
|
802
|
+
ch_treelen = [ (ch.edge.length + ch.treelen) for ch in children ]
|
|
803
|
+
nd.treelen = sum(ch_treelen)
|
|
804
|
+
ch_treelen_rank = np.argsort( ch_treelen )[::-1]
|
|
805
|
+
children_reordered = [ children[i] for i in ch_treelen_rank ]
|
|
806
|
+
nd.set_children(children_reordered)
|
|
807
|
+
|
|
808
|
+
# inorder traversal to fill matrix
|
|
809
|
+
phy.seed_node.edge.length = 0
|
|
810
|
+
for nd in phy.inorder_node_iter():
|
|
811
|
+
|
|
812
|
+
if nd.is_leaf():
|
|
813
|
+
heights[1,height_idx] = nd.edge.length
|
|
814
|
+
states[:,state_idx] = dat[nd.taxon.label].to_list()
|
|
815
|
+
state_idx += 1
|
|
816
|
+
else:
|
|
817
|
+
heights[0,height_idx] = nd.root_distance
|
|
818
|
+
heights[2,height_idx] = nd.edge.length
|
|
819
|
+
height_idx += 1
|
|
820
|
+
|
|
821
|
+
# stack the phylo and states tensors
|
|
822
|
+
if rescale:
|
|
823
|
+
heights = heights / np.max(heights)
|
|
824
|
+
phylo_tensor = np.vstack( [heights, states] )
|
|
825
|
+
|
|
826
|
+
return phylo_tensor
|
|
827
|
+
|
|
828
|
+
def encode_cblvs(self, phy, dat, tree_width, tree_encode_type, rescale=True):
|
|
829
|
+
"""
|
|
830
|
+
Encode Compact Bijective Ladderized Vector + States (CBLV+S) array
|
|
831
|
+
|
|
832
|
+
# num columns equals tree_width, 0-padding
|
|
833
|
+
# returns tensor with following rows
|
|
834
|
+
# 0: leaf node-to-last internal node distance
|
|
835
|
+
# 1: internal node root-distance
|
|
836
|
+
# 2: leaf node branch length
|
|
837
|
+
# 3: internal node branch ength
|
|
838
|
+
# 4+: state encoding
|
|
839
|
+
|
|
840
|
+
Arguments:
|
|
841
|
+
phy (dendropy.Tree): phylogenetic tree
|
|
842
|
+
dat (numpy.array): character data
|
|
843
|
+
tree_width (int): number of columns (max. num. taxa) in CPVS array
|
|
844
|
+
tree_encode_type (str): type of tree encoding ('height_only' or 'height_brlen')
|
|
845
|
+
rescale: set tree height to 1 before encoding when True
|
|
846
|
+
|
|
847
|
+
Returns:
|
|
848
|
+
numpy.ndarray: The encoded CDVs tensor.
|
|
849
|
+
"""
|
|
850
|
+
|
|
851
|
+
# data dimensions
|
|
852
|
+
num_char_row = dat.shape[0]
|
|
853
|
+
if tree_encode_type == 'height_only':
|
|
854
|
+
num_tree_row = 2
|
|
855
|
+
elif tree_encode_type == 'height_brlen':
|
|
856
|
+
num_tree_row = 4
|
|
857
|
+
|
|
858
|
+
|
|
859
|
+
# initialize workspace
|
|
860
|
+
null = phy.calc_node_root_distances(return_leaf_distances_only=False)
|
|
861
|
+
heights = np.zeros( (num_tree_row, tree_width) )
|
|
862
|
+
states = np.zeros( (num_char_row, tree_width) )
|
|
863
|
+
state_idx = 0
|
|
864
|
+
height_idx = 0
|
|
865
|
+
|
|
866
|
+
# postorder traversal to rotate nodes by max-root-distance
|
|
867
|
+
for nd in phy.postorder_node_iter():
|
|
868
|
+
if nd.is_leaf():
|
|
869
|
+
nd.max_root_distance = nd.root_distance
|
|
870
|
+
else:
|
|
871
|
+
children = nd.child_nodes()
|
|
872
|
+
ch_max_root_distance = [ ch.max_root_distance for ch in children ]
|
|
873
|
+
ch_max_root_distance_rank = np.argsort( ch_max_root_distance )[::-1] # [0,1] or [1,0]
|
|
874
|
+
children_reordered = [ children[i] for i in ch_max_root_distance_rank ]
|
|
875
|
+
nd.max_root_distance = max(ch_max_root_distance)
|
|
876
|
+
nd.set_children(children_reordered)
|
|
877
|
+
|
|
878
|
+
# inorder traversal to fill matrix
|
|
879
|
+
last_int_node = phy.seed_node
|
|
880
|
+
last_int_node.edge.length = 0
|
|
881
|
+
for nd in phy.inorder_node_iter():
|
|
882
|
+
if nd.is_leaf():
|
|
883
|
+
heights[0,height_idx] = nd.root_distance - last_int_node.root_distance
|
|
884
|
+
heights[2,height_idx] = nd.edge.length
|
|
885
|
+
states[:,state_idx] = dat[nd.taxon.label].to_list()
|
|
886
|
+
state_idx += 1
|
|
887
|
+
else:
|
|
888
|
+
#print(last_int_node.edge.length)
|
|
889
|
+
heights[1,height_idx+1] = nd.root_distance
|
|
890
|
+
heights[3,height_idx+1] = nd.edge.length
|
|
891
|
+
last_int_node = nd
|
|
892
|
+
height_idx += 1
|
|
893
|
+
|
|
894
|
+
# stack the phylo and states tensors
|
|
895
|
+
if rescale:
|
|
896
|
+
heights = heights / np.max(heights)
|
|
897
|
+
phylo_tensor = np.vstack( [heights, states] )
|
|
898
|
+
|
|
899
|
+
return phylo_tensor
|
|
900
|
+
|
|
901
|
+
#------------------------------------------------------------------------------#
|
|
902
|
+
|
|
903
|
+
|
|
904
|
+
# def write_tensor_hdf5_old(self):
|
|
905
|
+
# """
|
|
906
|
+
# Writes data to HDF5 file for each tree width.
|
|
907
|
+
|
|
908
|
+
# This class creates HDF5 files
|
|
909
|
+
|
|
910
|
+
# """
|
|
911
|
+
|
|
912
|
+
# # build files
|
|
913
|
+
# for tree_width in sorted(list(self.phy_tensors.keys())):
|
|
914
|
+
|
|
915
|
+
# # dimensions
|
|
916
|
+
# rep_idx = sorted(list(self.phy_tensors[tree_width]))
|
|
917
|
+
# num_samples = len(rep_idx)
|
|
918
|
+
# num_data_length = tree_width * self.num_data_row
|
|
919
|
+
|
|
920
|
+
# # print info
|
|
921
|
+
# print('Combining {n} files for tree_type={tt} and tree_width={ts}'.format(n=num_samples, tt=self.tree_encode, ts=tree_width))
|
|
922
|
+
|
|
923
|
+
# # HDF5 file
|
|
924
|
+
# out_hdf5_fn = f'{self.fmt_proj_dir}/sim.nt{tree_width}.hdf5'
|
|
925
|
+
# hdf5_file = h5py.File(out_hdf5_fn, 'w')
|
|
926
|
+
|
|
927
|
+
# # create datasets for numerical data
|
|
928
|
+
# dat_data = hdf5_file.create_dataset('phy_data',
|
|
929
|
+
# (num_samples, num_data_length),
|
|
930
|
+
# dtype='f', compression='gzip')
|
|
931
|
+
# dat_stat = hdf5_file.create_dataset('summ_stat',
|
|
932
|
+
# (num_samples, self.num_summ_stat),
|
|
933
|
+
# dtype='f', compression='gzip')
|
|
934
|
+
# dat_labels = hdf5_file.create_dataset('labels',
|
|
935
|
+
# (num_samples, self.num_labels),
|
|
936
|
+
# dtype='f', compression='gzip')
|
|
937
|
+
|
|
938
|
+
# # the replicates for this tree width
|
|
939
|
+
# _rep_idx = list(self.phy_tensors[tree_width].keys())
|
|
940
|
+
|
|
941
|
+
# # load all the info
|
|
942
|
+
# res = [ self.load_one_sim(idx=idx, tree_width=tree_width) for idx in tqdm(_rep_idx,
|
|
943
|
+
# total=len(_rep_idx),
|
|
944
|
+
# desc='Combining') ]
|
|
945
|
+
|
|
946
|
+
# # store all numerical data into hdf5)
|
|
947
|
+
# if len(res) > 0:
|
|
948
|
+
# dat_data[:,:] = np.vstack( [ x[0] for x in res ] )
|
|
949
|
+
# dat_stat[:,:] = np.vstack( [ x[1] for x in res ] )
|
|
950
|
+
# dat_labels[:,:] = np.vstack( [ x[2] for x in res ] )
|
|
951
|
+
|
|
952
|
+
# # read in summ_stats and labels (_all_ params) dataframes
|
|
953
|
+
# df_summ_stats = pd.DataFrame(dat_stat, columns=self.summ_stat_names)
|
|
954
|
+
# df_labels = pd.DataFrame(dat_labels, columns=self.label_names)
|
|
955
|
+
|
|
956
|
+
# # separate data parameters (things we know) from label parameters (things we estimate)
|
|
957
|
+
# df_labels_new = df_labels[self.param_est]
|
|
958
|
+
# df_labels_move = df_labels[self.param_data]
|
|
959
|
+
|
|
960
|
+
# # concatenate new data parameters as column to existing summ_stats dataframe
|
|
961
|
+
# df_aux_data = df_summ_stats.join( df_labels_move )
|
|
962
|
+
|
|
963
|
+
# # get new label/stat names
|
|
964
|
+
# new_label_names = self.param_est
|
|
965
|
+
# new_aux_data_names = self.summ_stat_names + self.param_data
|
|
966
|
+
|
|
967
|
+
# # delete original datasets
|
|
968
|
+
# del hdf5_file['summ_stat']
|
|
969
|
+
# del hdf5_file['labels']
|
|
970
|
+
|
|
971
|
+
# # create new datasets
|
|
972
|
+
# hdf5_file.create_dataset('labels', df_labels_new.shape, 'f', df_labels_new, compression='gzip')
|
|
973
|
+
# hdf5_file.create_dataset('label_names', (1, len(new_label_names)), 'S64', new_label_names, compression='gzip')
|
|
974
|
+
# hdf5_file.create_dataset('aux_data', df_aux_data.shape, 'f', df_aux_data, compression='gzip')
|
|
975
|
+
# hdf5_file.create_dataset('aux_data_names', (1, len(new_aux_data_names)), 'S64', new_aux_data_names, compression='gzip')
|
|
976
|
+
|
|
977
|
+
# # close HDF5 files
|
|
978
|
+
# hdf5_file.close()
|
|
979
|
+
|
|
980
|
+
# return
|
|
981
|
+
|
|
982
|
+
# def write_tensor_csv_old(self):
|
|
983
|
+
# """
|
|
984
|
+
# Writes CSV files for phylogenetic tensors.
|
|
985
|
+
|
|
986
|
+
# The method iterates through the phylogenetic tensors for each tree width
|
|
987
|
+
# and generates CSV files containing the tensor data and labels.
|
|
988
|
+
# """
|
|
989
|
+
# # build files
|
|
990
|
+
# for tree_width in sorted(list(self.phy_tensors.keys())):
|
|
991
|
+
|
|
992
|
+
# # helper variables
|
|
993
|
+
# phy_tensors = self.phy_tensors[tree_width]
|
|
994
|
+
# num_samples = len(phy_tensors)
|
|
995
|
+
|
|
996
|
+
# print('Formatting {n} files for tree_type={tt} and tree_width={ts}'.format(n=num_samples, tt=self.tree_encode, ts=tree_width))
|
|
997
|
+
|
|
998
|
+
# # output csv filepaths
|
|
999
|
+
# out_prefix = f'{self.fmt_proj_dir}/sim.nt{tree_width}'
|
|
1000
|
+
# in_prefix = f'{self.sim_proj_dir}/sim'
|
|
1001
|
+
# out_phys_fn = f'{out_prefix}.phy_data.csv'
|
|
1002
|
+
# out_stat_fn = f'{out_prefix}.aux_data.csv'
|
|
1003
|
+
# out_labels_fn = f'{out_prefix}.labels.csv'
|
|
1004
|
+
|
|
1005
|
+
# # phylogenetic state tensor
|
|
1006
|
+
# with open(out_phys_fn, 'w') as outfile:
|
|
1007
|
+
# for j,(idx,pt) in enumerate(phy_tensors.items()):
|
|
1008
|
+
# s = ','.join(map(str, pt.flatten())) + '\n'
|
|
1009
|
+
# outfile.write(s)
|
|
1010
|
+
|
|
1011
|
+
# # summary stats tensor
|
|
1012
|
+
# with open(out_stat_fn, 'w') as outfile:
|
|
1013
|
+
# for j,idx in enumerate(phy_tensors.keys()):
|
|
1014
|
+
# fname = f'{in_prefix}.{idx}.summ_stat.csv'
|
|
1015
|
+
# with open(fname, 'r') as infile:
|
|
1016
|
+
# if j == 0:
|
|
1017
|
+
# s = infile.read()
|
|
1018
|
+
# else:
|
|
1019
|
+
# s = ''.join(infile.readlines()[1:])
|
|
1020
|
+
# outfile.write(s)
|
|
1021
|
+
|
|
1022
|
+
# # labels input tensor
|
|
1023
|
+
# with open(out_labels_fn, 'w') as outfile:
|
|
1024
|
+
# for j,idx in enumerate(phy_tensors.keys()):
|
|
1025
|
+
# fname = f'{in_prefix}.{idx}.param_row.csv'
|
|
1026
|
+
# with open(fname, 'r') as infile:
|
|
1027
|
+
# if j == 0:
|
|
1028
|
+
# s = infile.read()
|
|
1029
|
+
# else:
|
|
1030
|
+
# s = ''.join(infile.readlines()[1:])
|
|
1031
|
+
# outfile.write(s)
|
|
1032
|
+
|
|
1033
|
+
# # rearrange labels and summary statistics
|
|
1034
|
+
# # - labels contains param_est
|
|
1035
|
+
# # - aux_data contains summ_stat and param_data
|
|
1036
|
+
|
|
1037
|
+
# # read in summ_stats and labels
|
|
1038
|
+
# df_summ_stats = pd.read_csv(out_stat_fn)
|
|
1039
|
+
# df_labels = pd.read_csv(out_labels_fn)
|
|
1040
|
+
|
|
1041
|
+
# # separate data parameters (things we know) from label parameters (things we predict)
|
|
1042
|
+
# df_labels_keep = df_labels[self.param_est]
|
|
1043
|
+
# df_labels_move = df_labels[self.param_data]
|
|
1044
|
+
|
|
1045
|
+
# # concatenate new data parameters as column to existing summ_stats dataframe
|
|
1046
|
+
# df_summ_stats = df_summ_stats.join( df_labels_move )
|
|
1047
|
+
|
|
1048
|
+
# # overwrite original files with new modified versions
|
|
1049
|
+
# df_summ_stats.to_csv(out_stat_fn, index=False)
|
|
1050
|
+
# df_labels_keep.to_csv(out_labels_fn, index=False)
|
|
1051
|
+
|
|
1052
|
+
# return
|