phyddle 0.0.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
phyddle/format.py ADDED
@@ -0,0 +1,1052 @@
1
+ #!/usr/bin/env python
2
+ """
3
+ format
4
+ ======
5
+ Defines classes and methods for the Format step. This step converts raw data
6
+ into tensor data that can be used by the Train step.
7
+
8
+ Authors: Michael Landis and Ammon Thompson
9
+ Copyright: (c) 2022-2023, Michael Landis and Ammon Thompson
10
+ License: MIT
11
+ """
12
+ # standard imports
13
+ import copy
14
+ import os
15
+ import sys
16
+
17
+ # external imports
18
+ import dendropy as dp
19
+ import h5py
20
+ import numpy as np
21
+ import scipy as sp
22
+ import pandas as pd
23
+ from multiprocessing import Pool, set_start_method, cpu_count
24
+ from tqdm import tqdm
25
+
26
+ # phyddle imports
27
+ from phyddle import utilities as util
28
+
29
+ # Uncomment to debug multiprocessing
30
+ # import multiprocessing.util as mp_util
31
+ # mp_util.log_to_stderr(mp_util.SUBDEBUG)
32
+
33
+ # Allows multiprocessing fork (not spawn) new processes on Unix-based OS.
34
+ # However, import phyddle.simulate also calls set_start_method('fork'), and the
35
+ # function throws RuntimeError when called the 2nd+ time within a single Python.
36
+ # We handle this with a try-except block.
37
+ try:
38
+ set_start_method('fork')
39
+ except RuntimeError:
40
+ pass
41
+
42
+ #------------------------------------------------------------------------------#
43
+
44
+ def load(args):
45
+ """
46
+ Load a Formatter object.
47
+
48
+ This function creates an instance of the Formatter class, initialized using
49
+ phyddle settings stored in args (dict).
50
+
51
+ Args:
52
+ args (dict): Contains phyddle settings.
53
+ """
54
+ # settings
55
+ sys.setrecursionlimit(10000)
56
+
57
+ # load object
58
+ format_method = 'default'
59
+ if format_method == 'default':
60
+ return Formatter(args)
61
+ else:
62
+ return NotImplementedError
63
+
64
+ #------------------------------------------------------------------------------#
65
+
66
+ class Formatter:
67
+ """
68
+ Class for formatting phylogenetic datasets and converting them into tensors
69
+ to be used by the Train step.
70
+ """
71
+ def __init__(self, args): #, mdl):
72
+ """
73
+ Initializes a new Formatter object.
74
+
75
+ Args:
76
+ args (dict): Contains phyddle settings.
77
+ """
78
+ # initialize with phyddle settings
79
+ self.set_args(args)
80
+ # directory for simulations (input)
81
+ self.sim_proj_dir = f'{self.sim_dir}/{self.sim_proj}'
82
+ # directory for formatted tensors (output)
83
+ self.fmt_proj_dir = f'{self.fmt_dir}/{self.fmt_proj}'
84
+ # set number of processors
85
+ if self.num_proc <= 0:
86
+ self.num_proc = cpu_count() + self.num_proc
87
+ # run() attempts to generate one simulation per value in rep_idx,
88
+ # where rep_idx is list of unique ints to identify simulated datasets
89
+ if self.encode_all_sim:
90
+ self.rep_idx = self.get_rep_idx()
91
+ else:
92
+ self.rep_idx = list(range(self.start_idx, self.end_idx))
93
+
94
+ # split_idx is later assigned a value, after raw data is encoded
95
+ self.split_idx = {}
96
+
97
+ # get size of CPV+S tensors
98
+ self.num_tree_row = util.get_num_tree_row(self.tree_encode,
99
+ self.brlen_encode)
100
+ self.num_char_row = util.get_num_char_row(self.char_encode,
101
+ self.num_char,
102
+ self.num_states)
103
+ self.num_data_row = self.num_tree_row + self.num_char_row
104
+
105
+ # create logger to track runtime info
106
+ self.logger = util.Logger(args)
107
+ # done
108
+ return
109
+
110
+ def set_args(self, args):
111
+ """
112
+ Assigns phyddle settings as Formatter attributes.
113
+
114
+ Args:
115
+ args (dict): Contains phyddle settings.
116
+
117
+ """
118
+ # formatter arguments
119
+ self.args = args
120
+ step_args = util.make_step_args('F', args)
121
+ for k,v in step_args.items():
122
+ setattr(self, k, v)
123
+
124
+ # special case
125
+ self.tree_width_cats = [ self.tree_width ] # will be removed
126
+
127
+ return
128
+
129
+ def run(self):
130
+ """
131
+ Formats all raw datasets into tensors.
132
+
133
+ This method prints status updates, creates the target directory for new
134
+ tensors, then formats each simulated raw dataset into an individual
135
+ tensor-formatted dataset, then combines all individual tensors into
136
+ a single tensor that contains all training examples.
137
+ """
138
+
139
+ verbose = self.verbose
140
+
141
+ # print header
142
+ util.print_step_header('fmt', [self.sim_proj_dir], self.fmt_proj_dir, verbose)
143
+
144
+ # prepare workspace
145
+ os.makedirs(self.fmt_proj_dir, exist_ok=True)
146
+
147
+ # start time
148
+ start_time,start_time_str = util.get_time()
149
+ util.print_str(f'▪ Start time of {start_time_str}', verbose)
150
+
151
+ # encode each dataset into individual tensors
152
+ util.print_str('▪ Encoding raw data as tensors', verbose)
153
+ self.encode_all()
154
+
155
+ # write tensors across all examples to file
156
+ util.print_str('▪ Combining and writing tensors', verbose)
157
+ self.write_tensor()
158
+
159
+ # end time
160
+ end_time,end_time_str = util.get_time()
161
+ run_time = util.get_time_diff(start_time, end_time)
162
+ # util.print_str(f'▪ End time: {end_time_str}', verbose)
163
+ util.print_str(f'▪ End time of {end_time_str} (+{run_time})', verbose)
164
+
165
+ # done
166
+ util.print_str('... done!', verbose)
167
+
168
+
169
+ def make_settings_str(self, idx, tree_width):
170
+ """
171
+ Construct a string of settings for a single replicate.
172
+
173
+ Args:
174
+ idx (int): The replicate index.
175
+ tree_width (int): The tree width.
176
+
177
+ Returns:
178
+ str: The settings string.
179
+ """
180
+ s = 'setting,value\n'
181
+ s += f'sim_proj_dir,{self.sim_proj_dir}\n'
182
+ s += f'fmt_proj_dir,{self.fmt_proj_dir}\n'
183
+ s += f'replicate_index,{idx}\n'
184
+ s += f'tree_width,{tree_width}\n'
185
+ return s
186
+
187
+ def encode_all(self):
188
+ """
189
+ Encode each simulated replicate into its own matrix format.
190
+
191
+ Encode each simulated dataset identified by the replicate-index list
192
+ (self.rep_idx). Each dataset is encoded by calling self.encode_one(idx)
193
+ where idx is a unique value in self.rep_idx.
194
+
195
+ Encoded simulations are then sorted by tree-width category and stored
196
+ into the phy_tensors dictionary for later processing. Names and lengths
197
+ of label and summary statistic lists are also saved.
198
+
199
+ When self.use_parallel is True then all jobs are run in parallel via
200
+ multiprocessing.Pool. When self.use_parallel is false, jobs are run
201
+ serially with one CPU.
202
+
203
+ """
204
+
205
+ # construct list of encoding arguments
206
+ args = []
207
+
208
+ for idx in self.rep_idx:
209
+ args.append((f'{self.sim_proj_dir}/sim.{idx}', idx))
210
+
211
+ # visit each replicate, encode it, and return result
212
+ if self.use_parallel:
213
+ # parallel jobs
214
+ with Pool(processes=self.num_proc) as pool:
215
+ # Note, it's critical to call this as list(tqdm(pool.imap(...)))
216
+ # - pool.imap runs the parallelization
217
+ # - tqdm generates progress bar
218
+ # - list acts as a finalizer for the pool.imap work
219
+ # Have not benchmarked imap/imap_unordered, chunksize, etc.
220
+ res = list(tqdm(pool.imap(self.encode_one_star,
221
+ args, chunksize=5),
222
+ total=len(args),
223
+ desc='Encoding',
224
+ smoothing=0))
225
+
226
+ else:
227
+ # serial jobs
228
+ res = [ self.encode_one_star(a) for a in tqdm(args,
229
+ total=len(args),
230
+ desc='Encoding',
231
+ smoothing=0) ]
232
+
233
+ # save all phylogenetic-state tensors into the phy_tensors dictionary,
234
+ # while sorting tensors into different tree-width categories
235
+ self.phy_tensors = {}
236
+ # for size in self.tree_width_cats:
237
+ # self.phy_tensors[size] = {}
238
+ for i in range(len(res)):
239
+ if res[i] is not None:
240
+ # tensor_size = res[i].shape[1]
241
+ # self.phy_tensors[tensor_size][i] = res[i]
242
+ self.phy_tensors[i] = res[i]
243
+
244
+ # save names/lengths of summary statistic and label lists
245
+ self.summ_stat_names = self.get_summ_stat_names()
246
+ self.label_names = self.get_label_names()
247
+ self.num_summ_stat = len(self.summ_stat_names)
248
+ self.num_labels = len(self.label_names)
249
+
250
+ return
251
+
252
+ def get_rep_idx(self):
253
+ rep_idx = set()
254
+ files = os.listdir(f'{self.sim_proj_dir}')
255
+ for f in files:
256
+ rep_idx.add(int(f.split('.')[1]))
257
+ rep_idx = sorted(list(rep_idx))
258
+ return rep_idx
259
+
260
+ def get_summ_stat_names(self):
261
+ """
262
+ Get names of summary statistics from first representative file.
263
+
264
+ Returns:
265
+ ret (list): List of summary statistics names.
266
+ """
267
+ # get first representative file
268
+ idx = None
269
+ # for i in self.tree_width_cats:
270
+ # k_list = list(self.phy_tensors[i].keys())
271
+ # if len(k_list) > 0 and idx is None:
272
+ # idx = k_list[0]
273
+ k_list = list(self.phy_tensors.keys())
274
+ if len(k_list) > 0 and idx is None:
275
+ idx = k_list[0]
276
+
277
+ fn = f'{self.sim_proj_dir}/sim.{idx}.summ_stat.csv'
278
+ df = pd.read_csv(fn,header=0)
279
+ ret = df.columns.to_list()
280
+ return ret
281
+
282
+ def get_label_names(self):
283
+ """
284
+ Get names of training labels from first representative file.
285
+
286
+ Returns:
287
+ ret (list): List of label names.
288
+ """
289
+ # get first representative file
290
+ idx = None
291
+ # for i in self.tree_width_cats:
292
+ # k_list = list(self.phy_tensors[i].keys())
293
+ # if len(k_list) > 0 and idx is None:
294
+ # idx = k_list[0]
295
+ k_list = list(self.phy_tensors.keys())
296
+ if len(k_list) > 0 and idx is None:
297
+ idx = k_list[0]
298
+
299
+ fn = f'{self.sim_proj_dir}/sim.{idx}.param_row.csv'
300
+ df = pd.read_csv(fn,header=0)
301
+ ret = df.columns.to_list()
302
+ return ret
303
+
304
+ def split_examples(self):
305
+ """
306
+ Split examples into training and test datasets.
307
+ """
308
+ split_idx = {}
309
+ # tree_width = self.tree_width_cats[0]
310
+ # rep_idx = sorted(list(self.phy_tensors[tree_width]))
311
+ rep_idx = sorted(list(self.phy_tensors.keys()))
312
+ num_samples = len(rep_idx)
313
+ np.random.shuffle(rep_idx)
314
+ num_test = int(num_samples * self.prop_test)
315
+ split_idx['test'] = rep_idx[:num_test]
316
+ split_idx['train'] = rep_idx[num_test:]
317
+
318
+ return split_idx
319
+
320
+ def write_tensor(self):
321
+ """
322
+ Write the tensor in csv or hdf5 format.
323
+ """
324
+ self.split_idx = self.split_examples()
325
+ if self.tensor_format == 'csv':
326
+ # self.write_tensor_csv()
327
+ self.write_tensor_csv('train')
328
+ self.write_tensor_csv('test')
329
+ elif self.tensor_format == 'hdf5':
330
+ # self.write_tensor_hdf5()
331
+ self.write_tensor_hdf5('train')
332
+ self.write_tensor_hdf5('test')
333
+ return
334
+
335
+
336
+ def write_tensor_hdf5(self, data_str):
337
+ """
338
+ Writes data to HDF5 file for each tree width.
339
+
340
+ This class creates HDF5 files
341
+
342
+ """
343
+
344
+ assert(data_str in ['test', 'train'])
345
+
346
+ # build files
347
+ tree_width = self.tree_width #self.tree_width_cats[0]
348
+ #phy_tensor = self.phy_tensors[tree_width]
349
+
350
+ # dimensions
351
+ rep_idx = self.split_idx[data_str]
352
+ num_samples = len(rep_idx)
353
+ num_data_length = tree_width * self.num_data_row
354
+
355
+ # print info
356
+ print(f'Making {data_str} hdf5 dataset: {num_samples} examples for tree width = {tree_width}')
357
+
358
+ # HDF5 file
359
+ out_hdf5_fn = f'{self.fmt_proj_dir}/{data_str}.nt{tree_width}.hdf5'
360
+ hdf5_file = h5py.File(out_hdf5_fn, 'w')
361
+
362
+ # create datasets for numerical data
363
+ dat_data = hdf5_file.create_dataset('phy_data',
364
+ (num_samples, num_data_length),
365
+ dtype='f', compression='gzip')
366
+ dat_stat = hdf5_file.create_dataset('summ_stat',
367
+ (num_samples, self.num_summ_stat),
368
+ dtype='f', compression='gzip')
369
+ dat_labels = hdf5_file.create_dataset('labels',
370
+ (num_samples, self.num_labels),
371
+ dtype='f', compression='gzip')
372
+
373
+ # the replicates for this tree width
374
+ #_rep_idx = list(phy_tensor.keys())
375
+
376
+ # load all the info
377
+ res = [ self.load_one_sim(idx=idx, tree_width=tree_width) for idx in tqdm(rep_idx,
378
+ total=len(rep_idx),
379
+ desc='Combining',
380
+ smoothing=0) ]
381
+
382
+ # store all numerical data into hdf5)
383
+ if len(res) > 0:
384
+ dat_data[:,:] = np.vstack( [ x[0] for x in res ] )
385
+ dat_stat[:,:] = np.vstack( [ x[1] for x in res ] )
386
+ dat_labels[:,:] = np.vstack( [ x[2] for x in res ] )
387
+
388
+ # read in summ_stats and labels (_all_ params) dataframes
389
+ df_summ_stats = pd.DataFrame(dat_stat, columns=self.summ_stat_names)
390
+ df_labels = pd.DataFrame(dat_labels, columns=self.label_names)
391
+
392
+ # separate data parameters (things we know) from label parameters (things we estimate)
393
+ df_labels_new = df_labels[self.param_est]
394
+ df_labels_move = df_labels[self.param_data]
395
+
396
+ # concatenate new data parameters as column to existing summ_stats dataframe
397
+ df_aux_data = df_summ_stats.join( df_labels_move )
398
+
399
+ # get new label/stat names
400
+ new_label_names = self.param_est
401
+ new_aux_data_names = self.summ_stat_names + self.param_data
402
+
403
+ # delete original datasets
404
+ del hdf5_file['summ_stat']
405
+ del hdf5_file['labels']
406
+
407
+ # create new datasets
408
+ hdf5_file.create_dataset('labels', df_labels_new.shape, 'f', df_labels_new, compression='gzip')
409
+ hdf5_file.create_dataset('label_names', (1, len(new_label_names)), 'S64', new_label_names, compression='gzip')
410
+ hdf5_file.create_dataset('aux_data', df_aux_data.shape, 'f', df_aux_data, compression='gzip')
411
+ hdf5_file.create_dataset('aux_data_names', (1, len(new_aux_data_names)), 'S64', new_aux_data_names, compression='gzip')
412
+
413
+ # close HDF5 files
414
+ hdf5_file.close()
415
+
416
+ return
417
+
418
+ def write_tensor_csv(self, data_str):
419
+ """
420
+ Writes CSV files for phylogenetic tensors.
421
+
422
+ The method iterates through the phylogenetic tensors for each tree width
423
+ and generates CSV files containing the tensor data and labels.
424
+ """
425
+ assert(data_str in ['test', 'train'])
426
+
427
+ # build files
428
+ tree_width = self.tree_width #_cats[0]
429
+ phy_tensor = self.phy_tensors #[tree_width]
430
+
431
+ # dimensions
432
+ rep_idx = self.split_idx[data_str]
433
+ num_samples = len(rep_idx)
434
+
435
+ # info
436
+ print(f'Making {data_str} csv dataset: {num_samples} examples for tree width = {tree_width}')
437
+
438
+ # output csv filepaths
439
+ #out_hdf5_fn = f'{self.fmt_proj_dir}/{data_str}.nt{tree_width}.hdf5'
440
+ out_prefix = f'{self.fmt_proj_dir}/{data_str}.nt{tree_width}'
441
+ in_prefix = f'{self.sim_proj_dir}/sim'
442
+ out_phys_fn = f'{out_prefix}.phy_data.csv'
443
+ out_stat_fn = f'{out_prefix}.aux_data.csv'
444
+ out_labels_fn = f'{out_prefix}.labels.csv'
445
+
446
+ # phylogenetic state tensor
447
+ with open(out_phys_fn, 'w') as outfile:
448
+ for idx in rep_idx:
449
+ pt = phy_tensor[idx]
450
+ #for j,(idx,pt) in enumerate(phy_tensor.items()):
451
+ s = ','.join(map(str, pt.flatten())) + '\n'
452
+ outfile.write(s)
453
+
454
+ # summary stats tensor
455
+ with open(out_stat_fn, 'w') as outfile:
456
+ is_first = True
457
+ for idx in rep_idx:
458
+ # for j,idx in enumerate(phy_tensor.keys()):
459
+ # if idx in rep_idx:
460
+ fname = f'{in_prefix}.{idx}.summ_stat.csv'
461
+ with open(fname, 'r') as infile:
462
+ if is_first:
463
+ s = infile.read()
464
+ is_first = False
465
+ else:
466
+ s = ''.join(infile.readlines()[1:])
467
+ outfile.write(s)
468
+
469
+ # labels input tensor
470
+ with open(out_labels_fn, 'w') as outfile:
471
+ is_first = True
472
+ for idx in rep_idx:
473
+ # for j,idx in enumerate(phy_tensor.keys()):
474
+ # if idx in rep_idx:
475
+ fname = f'{in_prefix}.{idx}.param_row.csv'
476
+ with open(fname, 'r') as infile:
477
+ if is_first:
478
+ s = infile.read()
479
+ is_first = False
480
+ else:
481
+ s = ''.join(infile.readlines()[1:])
482
+ outfile.write(s)
483
+
484
+ # rearrange labels and summary statistics
485
+ # - labels contains param_est
486
+ # - aux_data contains summ_stat and param_data
487
+
488
+ # read in summ_stats and labels
489
+ df_summ_stats = pd.read_csv(out_stat_fn)
490
+ df_labels = pd.read_csv(out_labels_fn)
491
+
492
+ # separate data parameters (things we know) from label parameters (things we predict)
493
+ df_labels_keep = df_labels[self.param_est]
494
+ df_labels_move = df_labels[self.param_data]
495
+
496
+ # concatenate new data parameters as column to existing summ_stats dataframe
497
+ df_summ_stats = df_summ_stats.join( df_labels_move )
498
+
499
+ # overwrite original files with new modified versions
500
+ df_summ_stats.to_csv(out_stat_fn, index=False)
501
+ df_labels_keep.to_csv(out_labels_fn, index=False)
502
+
503
+ return
504
+
505
+ def load_one_sim(self, idx, tree_width):
506
+ """Load data for one simulation given its index and tree width.
507
+
508
+ Args:
509
+ idx (int): Index of the simulation.
510
+ tree_width (int): Tree width for the simulation.
511
+
512
+ Returns:
513
+ Tuple of numpy arrays (x1, x2, x3).
514
+ """
515
+ fname_base = f'{self.sim_proj_dir}/sim.{idx}'
516
+ fname_param = fname_base + '.param_row.csv'
517
+ fname_stat = fname_base + '.summ_stat.csv'
518
+ # x1 = self.phy_tensors[tree_width][idx].flatten()
519
+ x1 = self.phy_tensors[idx].flatten()
520
+ x2 = np.loadtxt(fname_stat, delimiter=',', skiprows=1)
521
+ x3 = np.loadtxt(fname_param, delimiter=',', skiprows=1)
522
+ return (x1,x2,x3)
523
+
524
+ def encode_one_star(self, args):
525
+ """Wrapper for encode_one w/ unpacked args"""
526
+ return self.encode_one(*args)
527
+
528
+ def encode_one(self, tmp_fn, idx, save_phyenc_csv=False):
529
+ """
530
+ Encode a single simulated raw dataset into tensor format.
531
+
532
+ This function transforms raw input into tensor outputs. The inputs are
533
+ read from a tree file and data matrix file. The tree is filtered to
534
+ target ranges of taxon counts and pruned of non-extant taxa.
535
+
536
+ Trees are then binned into tree-width categories (number of columns for
537
+ compact vector representation). Next, trees and character data are
538
+ encoded into extended CBLV or CDV formats that contain additional state
539
+ and branch length information.
540
+
541
+ Arguments:
542
+ tmp_fn (str): prefix for individual dataset
543
+ idx (int): replicate index for simulation
544
+ save_phyenc_csv (bool): save phylogenetic state tensor as csv?
545
+
546
+ Returns:
547
+ cpvs (numpy.array): compact phylo vector + states (CPVS)
548
+ """
549
+ NUM_DIGITS = 10
550
+ np.set_printoptions(formatter={'float': lambda x: format(x, '8.6E')},
551
+ precision=NUM_DIGITS)
552
+
553
+ # make filenames
554
+ dat_nex_fn = tmp_fn + '.dat.nex'
555
+ tre_fn = tmp_fn + '.tre'
556
+ prune_fn = tmp_fn + '.extant.tre'
557
+ down_fn = tmp_fn + '.downsampled.tre'
558
+ cpsv_fn = tmp_fn + '.phy_data.csv'
559
+ ss_fn = tmp_fn + '.summ_stat.csv'
560
+ info_fn = tmp_fn + '.info.csv'
561
+
562
+ # check if key files exist
563
+ err_msg = None
564
+ if not os.path.exists(dat_nex_fn):
565
+ err_msg = f'Formatter.encode_one(): {dat_nex_fn} does not exist'
566
+ print(err_msg)
567
+ self.logger.write_log('fmt', err_msg)
568
+ if not os.path.exists(tre_fn):
569
+ err_msg = f'Formatter.encode_one(): {tre_fn} does not exist'
570
+ print(err_msg)
571
+ self.logger.write_log('fmt', err_msg)
572
+ if err_msg is not None:
573
+ return
574
+
575
+ # read in nexus data file as numpy array
576
+ if self.char_format == 'nexus':
577
+ dat = util.convert_nexus_to_array(dat_nex_fn,
578
+ self.char_encode,
579
+ self.num_states)
580
+ elif self.char_format == 'csv':
581
+ dat = util.convert_csv_to_array(dat_nex_fn,
582
+ self.char_encode,
583
+ self.num_states)
584
+
585
+ # get tree file
586
+ phy = util.read_tree(tre_fn)
587
+ if phy is None:
588
+ return
589
+
590
+ # prune tree, if needed
591
+ if self.tree_encode == 'extant':
592
+ phy_prune = util.make_prune_phy(phy, prune_fn)
593
+ if phy_prune is None:
594
+ # abort, no valid pruned tree
595
+ return
596
+ else:
597
+ # valid pruned tree
598
+ phy = copy.deepcopy(phy_prune)
599
+
600
+ # downsample taxa
601
+ num_taxa_orig = len(phy.leaf_nodes())
602
+ phy = util.make_downsample_phy(
603
+ phy, down_fn,
604
+ max_taxa=max(self.tree_width_cats),
605
+ strategy=self.downsample_taxa)
606
+
607
+ # get tree size
608
+ num_taxa = len(phy.leaf_nodes())
609
+ if num_taxa > np.max(self.tree_width_cats):
610
+ # abort, too many taxa
611
+ return
612
+ if num_taxa < self.min_num_taxa or num_taxa < 0:
613
+ # abort, too few taxa
614
+ return
615
+
616
+ # get tree width from resulting vector
617
+ tree_width = util.find_tree_width(num_taxa, self.tree_width_cats)
618
+
619
+ # create compact phylo-state vector, CPV+S = {CBLV+S, CDV+S}
620
+ cpvs_data = None
621
+
622
+ # encode CBLV+S
623
+ cpvs_data = self.encode_cpvs(phy, dat, tree_width=tree_width,
624
+ tree_encode_type=self.brlen_encode,
625
+ tree_type=self.tree_encode)
626
+
627
+ # save CPVS
628
+ save_phyenc_csv_ = self.save_phyenc_csv or save_phyenc_csv
629
+ if save_phyenc_csv_ and cpvs_data is not None:
630
+ cpsv_str = util.make_clean_phyloenc_str(cpvs_data.flatten())
631
+ util.write_to_file(cpsv_str, cpsv_fn)
632
+
633
+ # record info
634
+ info_str = self.make_settings_str(idx, tree_width)
635
+ util.write_to_file(info_str, info_fn)
636
+
637
+ # record summ stat data
638
+ ss = self.make_summ_stat(phy, dat)
639
+ # add downsampling info
640
+ ss['num_taxa'] = num_taxa_orig
641
+ ss['prop_taxa'] = num_taxa / num_taxa_orig
642
+
643
+ # save summ. stats.
644
+ ss_str = self.make_summ_stat_str(ss)
645
+ util.write_to_file(ss_str, ss_fn)
646
+
647
+ # done!
648
+ return cpvs_data
649
+
650
+ def make_summ_stat(self, phy, dat):
651
+ """
652
+ Generate summary statistics.
653
+
654
+ This function populates a dictionary of summary statistics. Keys
655
+ are the names of statistics that later appear as column names in the
656
+ summ_stat and aux_data files and containers. Values are for the
657
+ inputted phy and dat variables.
658
+
659
+ Arguments:
660
+ phy (dendropy.Tree): phylogenetic tree
661
+ dat (numpy.array): character data
662
+
663
+ Returns:
664
+ summ_stats (dict): summary statistics
665
+ """
666
+ # new dictionary to return
667
+ summ_stats = {}
668
+
669
+ # read basic info from phylogenetic tree
670
+ num_taxa = len(phy.leaf_nodes())
671
+ node_ages = phy.internal_node_ages(ultrametricity_precision=False)
672
+ root_age = phy.seed_node.age
673
+ branch_lengths = [ nd.edge.length for nd in phy.nodes() if nd != phy.seed_node ]
674
+ #root_distances = phy.calc_node_root_distances()
675
+ #root_distances = [ nd.root_distance for nd in phy.nodes() if nd.is_leaf]
676
+ #phy.calc_node_ages(ultrametricity_precision=False)
677
+ #tree_height = np.max( root_distances )
678
+
679
+ # tree statistics
680
+ summ_stats['tree_length'] = phy.length()
681
+ summ_stats['root_age'] = root_age
682
+ summ_stats['brlen_mean'] = np.mean(branch_lengths)
683
+ summ_stats['brlen_var'] = np.var(branch_lengths)
684
+ summ_stats['brlen_skew'] = sp.stats.skew(branch_lengths)
685
+ summ_stats['age_mean'] = np.mean(node_ages)
686
+ summ_stats['age_var'] = np.var(node_ages)
687
+ summ_stats['age_skew'] = sp.stats.skew(node_ages)
688
+ summ_stats['B1'] = dp.calculate.treemeasure.B1(phy)
689
+ summ_stats['N_bar'] = dp.calculate.treemeasure.N_bar(phy)
690
+ summ_stats['colless'] = dp.calculate.treemeasure.colless_tree_imbalance(phy)
691
+ summ_stats['treeness'] = dp.calculate.treemeasure.treeness(phy)
692
+
693
+ # possible tree statistics, but not computable for arbitrary trees
694
+ #summ_stats['gamma'] = dp.calculate.treemeasure.pybus_harvey_gamma(phy)
695
+ #summ_stats['brlen_kurt'] = sp.stats.kurtosis(branch_lengths)
696
+ #summ_stats['age_kurt'] = sp.stats.kurtosis(root_distances)
697
+ #summ_stats['sackin'] = dp.calculate.treemeasure.sackin_index(phy)
698
+
699
+ # frequencies of character states
700
+ if self.char_encode == 'integer':
701
+ # integer-encoded states
702
+ for i in range(self.num_states):
703
+ summ_stats['f_dat_' + str(i)] = 0
704
+ unique, counts = np.unique(dat, return_counts=True)
705
+ for i,j in zip(unique, counts):
706
+ summ_stats['f_dat_' + str(i)] = j / num_taxa
707
+
708
+ elif self.char_encode == 'one_hot':
709
+ # one-hot-encoded states
710
+ for i in range(dat.shape[0]):
711
+ summ_stats['f_dat_' + str(i)] = np.sum(dat.iloc[i]) / num_taxa
712
+
713
+ # done
714
+ return summ_stats
715
+
716
+ # ==> Can probably move to util? seems generic and useful
717
+ def make_summ_stat_str(self, ss):
718
+ """
719
+ Generate a string representation of the summary statistics.
720
+
721
+ Arguments:
722
+ ss (dict): dictionary of summary statistics
723
+
724
+ Returns:
725
+ keys_str: string containing the keys of the summary statistics
726
+ vals_str: string containing the values of the summary statistics
727
+
728
+ """
729
+ keys_str = ','.join( list(ss.keys()) ) + '\n'
730
+ vals_str = ','.join( [ str(x) for x in ss.values() ] ) + '\n'
731
+ return keys_str + vals_str
732
+
733
+ def encode_cpvs(self, phy, dat, tree_width, tree_type, tree_encode_type, rescale=True):
734
+ """
735
+ Encode Compact Phylogenetic Vector + States (CPV+S) array
736
+
737
+ This function encodes the dataset into Compact Bijective Ladderized
738
+ Vector + States (CBLV+S) when tree_type is 'serial' or Compact
739
+ Diversity-Reordered Vector + States (CDV+S) when tree_type is 'extant'.
740
+
741
+ Arguments:
742
+ phy (dendropy.Tree): phylogenetic tree
743
+ dat (numpy.array): character data
744
+ tree_width (int): number of columns (max. num. taxa) in CPVS array
745
+ tree_type (str): type of the tree ('serial' or 'extant')
746
+ tree_encode_type (str): type of tree encoding ('height_only' or 'height_brlen')
747
+ rescale: set tree height to 1 before encoding when True
748
+
749
+ Returns:
750
+ cpvs (numpy.array): CPV+S encoded tensor
751
+ """
752
+ if tree_type == 'serial':
753
+ cpvs = self.encode_cblvs(phy, dat, tree_width, tree_encode_type, rescale)
754
+ elif tree_type == 'extant':
755
+ cpvs = self.encode_cdvs(phy, dat, tree_width, tree_encode_type, rescale)
756
+ else:
757
+ ValueError(f'Unrecognized {tree_type}')
758
+ return cpvs
759
+
760
+ def encode_cdvs(self, phy, dat, tree_width, tree_encode_type, rescale=True):
761
+ """
762
+ Encode Compact Diversity-reordered Vector + States (CDV+S) array
763
+
764
+ # num columns equals tree_width, 0-padding
765
+ # returns tensor with following rows
766
+ # 0: internal node root-distance
767
+ # 1: leaf node branch length
768
+ # 2: internal node branch ength
769
+ # 3+: state encoding
770
+
771
+ Arguments:
772
+ phy (dendropy.Tree): phylogenetic tree
773
+ dat (numpy.array): character data
774
+ tree_width (int): number of columns (max. num. taxa) in CPVS array
775
+ tree_encode_type (str): type of tree encoding ('height_only' or 'height_brlen')
776
+ rescale: set tree height to 1 before encoding when True
777
+
778
+ Returns:
779
+ numpy.ndarray: The encoded CDVs tensor.
780
+ """
781
+
782
+ # data dimensions
783
+ num_char_row = dat.shape[0]
784
+ if tree_encode_type == 'height_only':
785
+ num_tree_row = 1
786
+ elif tree_encode_type == 'height_brlen':
787
+ num_tree_row = 3
788
+
789
+ # initialize workspace
790
+ root_distances = phy.calc_node_root_distances(return_leaf_distances_only=False)
791
+ heights = np.zeros( (num_tree_row, tree_width) )
792
+ states = np.zeros( (num_char_row, tree_width) )
793
+ state_idx = 0
794
+ height_idx = 0
795
+
796
+ # postorder traversal to rotate nodes by clade-length
797
+ for nd in phy.postorder_node_iter():
798
+ if nd.is_leaf():
799
+ nd.treelen = 0.
800
+ else:
801
+ children = nd.child_nodes()
802
+ ch_treelen = [ (ch.edge.length + ch.treelen) for ch in children ]
803
+ nd.treelen = sum(ch_treelen)
804
+ ch_treelen_rank = np.argsort( ch_treelen )[::-1]
805
+ children_reordered = [ children[i] for i in ch_treelen_rank ]
806
+ nd.set_children(children_reordered)
807
+
808
+ # inorder traversal to fill matrix
809
+ phy.seed_node.edge.length = 0
810
+ for nd in phy.inorder_node_iter():
811
+
812
+ if nd.is_leaf():
813
+ heights[1,height_idx] = nd.edge.length
814
+ states[:,state_idx] = dat[nd.taxon.label].to_list()
815
+ state_idx += 1
816
+ else:
817
+ heights[0,height_idx] = nd.root_distance
818
+ heights[2,height_idx] = nd.edge.length
819
+ height_idx += 1
820
+
821
+ # stack the phylo and states tensors
822
+ if rescale:
823
+ heights = heights / np.max(heights)
824
+ phylo_tensor = np.vstack( [heights, states] )
825
+
826
+ return phylo_tensor
827
+
828
+ def encode_cblvs(self, phy, dat, tree_width, tree_encode_type, rescale=True):
829
+ """
830
+ Encode Compact Bijective Ladderized Vector + States (CBLV+S) array
831
+
832
+ # num columns equals tree_width, 0-padding
833
+ # returns tensor with following rows
834
+ # 0: leaf node-to-last internal node distance
835
+ # 1: internal node root-distance
836
+ # 2: leaf node branch length
837
+ # 3: internal node branch ength
838
+ # 4+: state encoding
839
+
840
+ Arguments:
841
+ phy (dendropy.Tree): phylogenetic tree
842
+ dat (numpy.array): character data
843
+ tree_width (int): number of columns (max. num. taxa) in CPVS array
844
+ tree_encode_type (str): type of tree encoding ('height_only' or 'height_brlen')
845
+ rescale: set tree height to 1 before encoding when True
846
+
847
+ Returns:
848
+ numpy.ndarray: The encoded CDVs tensor.
849
+ """
850
+
851
+ # data dimensions
852
+ num_char_row = dat.shape[0]
853
+ if tree_encode_type == 'height_only':
854
+ num_tree_row = 2
855
+ elif tree_encode_type == 'height_brlen':
856
+ num_tree_row = 4
857
+
858
+
859
+ # initialize workspace
860
+ null = phy.calc_node_root_distances(return_leaf_distances_only=False)
861
+ heights = np.zeros( (num_tree_row, tree_width) )
862
+ states = np.zeros( (num_char_row, tree_width) )
863
+ state_idx = 0
864
+ height_idx = 0
865
+
866
+ # postorder traversal to rotate nodes by max-root-distance
867
+ for nd in phy.postorder_node_iter():
868
+ if nd.is_leaf():
869
+ nd.max_root_distance = nd.root_distance
870
+ else:
871
+ children = nd.child_nodes()
872
+ ch_max_root_distance = [ ch.max_root_distance for ch in children ]
873
+ ch_max_root_distance_rank = np.argsort( ch_max_root_distance )[::-1] # [0,1] or [1,0]
874
+ children_reordered = [ children[i] for i in ch_max_root_distance_rank ]
875
+ nd.max_root_distance = max(ch_max_root_distance)
876
+ nd.set_children(children_reordered)
877
+
878
+ # inorder traversal to fill matrix
879
+ last_int_node = phy.seed_node
880
+ last_int_node.edge.length = 0
881
+ for nd in phy.inorder_node_iter():
882
+ if nd.is_leaf():
883
+ heights[0,height_idx] = nd.root_distance - last_int_node.root_distance
884
+ heights[2,height_idx] = nd.edge.length
885
+ states[:,state_idx] = dat[nd.taxon.label].to_list()
886
+ state_idx += 1
887
+ else:
888
+ #print(last_int_node.edge.length)
889
+ heights[1,height_idx+1] = nd.root_distance
890
+ heights[3,height_idx+1] = nd.edge.length
891
+ last_int_node = nd
892
+ height_idx += 1
893
+
894
+ # stack the phylo and states tensors
895
+ if rescale:
896
+ heights = heights / np.max(heights)
897
+ phylo_tensor = np.vstack( [heights, states] )
898
+
899
+ return phylo_tensor
900
+
901
+ #------------------------------------------------------------------------------#
902
+
903
+
904
+ # def write_tensor_hdf5_old(self):
905
+ # """
906
+ # Writes data to HDF5 file for each tree width.
907
+
908
+ # This class creates HDF5 files
909
+
910
+ # """
911
+
912
+ # # build files
913
+ # for tree_width in sorted(list(self.phy_tensors.keys())):
914
+
915
+ # # dimensions
916
+ # rep_idx = sorted(list(self.phy_tensors[tree_width]))
917
+ # num_samples = len(rep_idx)
918
+ # num_data_length = tree_width * self.num_data_row
919
+
920
+ # # print info
921
+ # print('Combining {n} files for tree_type={tt} and tree_width={ts}'.format(n=num_samples, tt=self.tree_encode, ts=tree_width))
922
+
923
+ # # HDF5 file
924
+ # out_hdf5_fn = f'{self.fmt_proj_dir}/sim.nt{tree_width}.hdf5'
925
+ # hdf5_file = h5py.File(out_hdf5_fn, 'w')
926
+
927
+ # # create datasets for numerical data
928
+ # dat_data = hdf5_file.create_dataset('phy_data',
929
+ # (num_samples, num_data_length),
930
+ # dtype='f', compression='gzip')
931
+ # dat_stat = hdf5_file.create_dataset('summ_stat',
932
+ # (num_samples, self.num_summ_stat),
933
+ # dtype='f', compression='gzip')
934
+ # dat_labels = hdf5_file.create_dataset('labels',
935
+ # (num_samples, self.num_labels),
936
+ # dtype='f', compression='gzip')
937
+
938
+ # # the replicates for this tree width
939
+ # _rep_idx = list(self.phy_tensors[tree_width].keys())
940
+
941
+ # # load all the info
942
+ # res = [ self.load_one_sim(idx=idx, tree_width=tree_width) for idx in tqdm(_rep_idx,
943
+ # total=len(_rep_idx),
944
+ # desc='Combining') ]
945
+
946
+ # # store all numerical data into hdf5)
947
+ # if len(res) > 0:
948
+ # dat_data[:,:] = np.vstack( [ x[0] for x in res ] )
949
+ # dat_stat[:,:] = np.vstack( [ x[1] for x in res ] )
950
+ # dat_labels[:,:] = np.vstack( [ x[2] for x in res ] )
951
+
952
+ # # read in summ_stats and labels (_all_ params) dataframes
953
+ # df_summ_stats = pd.DataFrame(dat_stat, columns=self.summ_stat_names)
954
+ # df_labels = pd.DataFrame(dat_labels, columns=self.label_names)
955
+
956
+ # # separate data parameters (things we know) from label parameters (things we estimate)
957
+ # df_labels_new = df_labels[self.param_est]
958
+ # df_labels_move = df_labels[self.param_data]
959
+
960
+ # # concatenate new data parameters as column to existing summ_stats dataframe
961
+ # df_aux_data = df_summ_stats.join( df_labels_move )
962
+
963
+ # # get new label/stat names
964
+ # new_label_names = self.param_est
965
+ # new_aux_data_names = self.summ_stat_names + self.param_data
966
+
967
+ # # delete original datasets
968
+ # del hdf5_file['summ_stat']
969
+ # del hdf5_file['labels']
970
+
971
+ # # create new datasets
972
+ # hdf5_file.create_dataset('labels', df_labels_new.shape, 'f', df_labels_new, compression='gzip')
973
+ # hdf5_file.create_dataset('label_names', (1, len(new_label_names)), 'S64', new_label_names, compression='gzip')
974
+ # hdf5_file.create_dataset('aux_data', df_aux_data.shape, 'f', df_aux_data, compression='gzip')
975
+ # hdf5_file.create_dataset('aux_data_names', (1, len(new_aux_data_names)), 'S64', new_aux_data_names, compression='gzip')
976
+
977
+ # # close HDF5 files
978
+ # hdf5_file.close()
979
+
980
+ # return
981
+
982
+ # def write_tensor_csv_old(self):
983
+ # """
984
+ # Writes CSV files for phylogenetic tensors.
985
+
986
+ # The method iterates through the phylogenetic tensors for each tree width
987
+ # and generates CSV files containing the tensor data and labels.
988
+ # """
989
+ # # build files
990
+ # for tree_width in sorted(list(self.phy_tensors.keys())):
991
+
992
+ # # helper variables
993
+ # phy_tensors = self.phy_tensors[tree_width]
994
+ # num_samples = len(phy_tensors)
995
+
996
+ # print('Formatting {n} files for tree_type={tt} and tree_width={ts}'.format(n=num_samples, tt=self.tree_encode, ts=tree_width))
997
+
998
+ # # output csv filepaths
999
+ # out_prefix = f'{self.fmt_proj_dir}/sim.nt{tree_width}'
1000
+ # in_prefix = f'{self.sim_proj_dir}/sim'
1001
+ # out_phys_fn = f'{out_prefix}.phy_data.csv'
1002
+ # out_stat_fn = f'{out_prefix}.aux_data.csv'
1003
+ # out_labels_fn = f'{out_prefix}.labels.csv'
1004
+
1005
+ # # phylogenetic state tensor
1006
+ # with open(out_phys_fn, 'w') as outfile:
1007
+ # for j,(idx,pt) in enumerate(phy_tensors.items()):
1008
+ # s = ','.join(map(str, pt.flatten())) + '\n'
1009
+ # outfile.write(s)
1010
+
1011
+ # # summary stats tensor
1012
+ # with open(out_stat_fn, 'w') as outfile:
1013
+ # for j,idx in enumerate(phy_tensors.keys()):
1014
+ # fname = f'{in_prefix}.{idx}.summ_stat.csv'
1015
+ # with open(fname, 'r') as infile:
1016
+ # if j == 0:
1017
+ # s = infile.read()
1018
+ # else:
1019
+ # s = ''.join(infile.readlines()[1:])
1020
+ # outfile.write(s)
1021
+
1022
+ # # labels input tensor
1023
+ # with open(out_labels_fn, 'w') as outfile:
1024
+ # for j,idx in enumerate(phy_tensors.keys()):
1025
+ # fname = f'{in_prefix}.{idx}.param_row.csv'
1026
+ # with open(fname, 'r') as infile:
1027
+ # if j == 0:
1028
+ # s = infile.read()
1029
+ # else:
1030
+ # s = ''.join(infile.readlines()[1:])
1031
+ # outfile.write(s)
1032
+
1033
+ # # rearrange labels and summary statistics
1034
+ # # - labels contains param_est
1035
+ # # - aux_data contains summ_stat and param_data
1036
+
1037
+ # # read in summ_stats and labels
1038
+ # df_summ_stats = pd.read_csv(out_stat_fn)
1039
+ # df_labels = pd.read_csv(out_labels_fn)
1040
+
1041
+ # # separate data parameters (things we know) from label parameters (things we predict)
1042
+ # df_labels_keep = df_labels[self.param_est]
1043
+ # df_labels_move = df_labels[self.param_data]
1044
+
1045
+ # # concatenate new data parameters as column to existing summ_stats dataframe
1046
+ # df_summ_stats = df_summ_stats.join( df_labels_move )
1047
+
1048
+ # # overwrite original files with new modified versions
1049
+ # df_summ_stats.to_csv(out_stat_fn, index=False)
1050
+ # df_labels_keep.to_csv(out_labels_fn, index=False)
1051
+
1052
+ # return