phyddle 0.0.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
phyddle/__init__.py ADDED
@@ -0,0 +1,33 @@
1
+ #!/usr/bin/env python
2
+ """
3
+ __init__
4
+ ===========
5
+ Initializes phyddle package and discovers all modules.
6
+
7
+ Authors: Michael Landis, Ammon Thompson
8
+ Copyright: (c) 2022-2023, Michael Landis and Ammon Thompson
9
+ License: MIT
10
+ """
11
+
12
+ #import os
13
+ #import sys
14
+ from multiprocessing import set_start_method
15
+
16
+ __project__ = 'phyddle'
17
+ __version__ = '0.0.7'
18
+ __author__ = 'Michael Landis and Ammon Thompson'
19
+ __copyright__ = '(c) 2022-2023, Michael Landis and Ammon Thompson'
20
+ __citation__ = 'A Thompson, B Liebeskind, EJ Scully, MJ Landis. 2023. Deep learning approaches to viral phylogeography are fast and as robust as likelihood methods to model misspecification. bioRxiv 2023.02.08.527714.'
21
+
22
+ # DEFAULT
23
+ CONFIG_DEFAULT_FN = '__config_default.py'
24
+ PHYDDLE_VERSION = __version__
25
+
26
+ # add local directory to system path
27
+ # allows import of local cfg
28
+ # ...but might switch over to simple tab-deilmited file
29
+ #sys.path.insert(0, os.getcwd())
30
+
31
+ if __name__ == '__main__':
32
+ # set the start method
33
+ set_start_method('fork')
phyddle/__main__.py ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env python3
2
+ from phyddle.command_line import run
3
+
4
+ def main():
5
+ run()
6
+
7
+ if __name__ == '__main__':
8
+ main()
@@ -0,0 +1,53 @@
1
+ #!/usr/bin/env python3
2
+
3
+ def run():
4
+
5
+ ########################
6
+ # LOAD PIPELINE CONFIG #
7
+ ########################
8
+
9
+ import phyddle.utilities as util
10
+ my_args = util.load_config('config', arg_overwrite=True)
11
+ step = my_args['step']
12
+
13
+ logger = util.Logger(my_args)
14
+ logger.save_log('run')
15
+
16
+ ################
17
+ # RUN PIPELINE #
18
+ ################
19
+
20
+ # Step 1: simulate training data
21
+ if 'S' in step:
22
+ import phyddle.simulate as sim
23
+ my_sim = sim.load(my_args)
24
+ my_sim.run()
25
+
26
+ # Step 2: format training data into tensors
27
+ if 'F' in step:
28
+ import phyddle.format as fmt
29
+ my_fmt = fmt.load(my_args)
30
+ my_fmt.run()
31
+
32
+ # Step 3: train network with training data
33
+ if 'T' in step:
34
+ import phyddle.train as trn
35
+ my_trn = trn.load(my_args)
36
+ my_trn.run()
37
+
38
+ # Step 4: estimates for new dataset
39
+ if 'E' in step:
40
+ import phyddle.format as fmt
41
+ import phyddle.estimate as est
42
+ est_prefix = f"{my_args['est_dir']}/{my_args['proj']}/{my_args['est_prefix']}"
43
+ my_fmt = fmt.load(my_args)
44
+ my_fmt.encode_one(tmp_fn=est_prefix, idx=-1, save_phyenc_csv=True)
45
+ my_est = est.load(my_args)
46
+ my_est.run()
47
+
48
+ # Step 5: plot results
49
+ if 'P' in step:
50
+ import phyddle.plot as plt
51
+ my_plt = plt.load(my_args)
52
+ my_plt.run()
53
+
phyddle/estimate.py ADDED
@@ -0,0 +1,409 @@
1
+ #!/usr/bin/env python
2
+ """
3
+ estimate
4
+ ========
5
+ Defines classes and methods for the Estimate step, which loads a pre-trained
6
+ network and uses it to generate new estimates, e.g. estimate model parmaeters
7
+ for a new empirical dataset.
8
+
9
+ Authors: Michael Landis and Ammon Thompson
10
+ Copyright: (c) 2022-2023, Michael Landis and Ammon Thompson
11
+ License: MIT
12
+ """
13
+
14
+ # standard imports
15
+ import os
16
+
17
+ # external imports
18
+ import numpy as np
19
+ import pandas as pd
20
+ import tensorflow as tf
21
+ import h5py
22
+
23
+ # phyddle imports
24
+ from phyddle import utilities as util
25
+
26
+ #------------------------------------------------------------------------------#
27
+
28
+ def load(args):
29
+ """
30
+ Load an Estimator object.
31
+
32
+ This function creates an instance of the Estimator class, initialized using
33
+ phyddle settings stored in args (dict).
34
+
35
+ Args:
36
+ args (dict): Contains phyddle settings.
37
+ """
38
+
39
+ # load object
40
+ est_method = 'default'
41
+ if est_method == 'default':
42
+ return Estimator(args)
43
+ else:
44
+ return NotImplementedError
45
+
46
+ #------------------------------------------------------------------------------#
47
+
48
+ class Estimator:
49
+ """
50
+ Class for making neural network estimates (i.e. label predictions) from new
51
+ (e.g. empirical) phylogenetic datasets. This class requires a trained
52
+ network from Train and input processed by Format. Output is written to
53
+ file and can be visualized using Plot.
54
+ """
55
+
56
+ def __init__(self, args):
57
+ """
58
+ Initializes a new Simulator object.
59
+
60
+ Args:
61
+ args (dict): Contains phyddle settings.
62
+ """
63
+ # initialize with phyddle settings
64
+ self.set_args(args)
65
+ # construct filepaths
66
+ self.prepare_filepaths()
67
+ # get size of CPV+S tensors
68
+ self.num_tree_row = util.get_num_tree_row(self.tree_encode,
69
+ self.brlen_encode)
70
+ self.num_char_row = util.get_num_char_row(self.char_encode,
71
+ self.num_char,
72
+ self.num_states)
73
+ self.num_data_row = self.num_tree_row + self.num_char_row
74
+ # create logger to track runtime info
75
+ self.logger = util.Logger(args)
76
+ # done
77
+ return
78
+
79
+ def set_args(self, args):
80
+ """
81
+ Assigns phyddle settings as Estimator attributes.
82
+
83
+ Args:
84
+ args (dict): Contains phyddle settings.
85
+ """
86
+ # estimator arguments
87
+ self.args = args
88
+ step_args = util.make_step_args('E', args)
89
+ for k,v in step_args.items():
90
+ setattr(self, k, v)
91
+
92
+ # self.args = args
93
+ # self.verbose = args['verbose']
94
+ # self.trn_dir = args['trn_dir']
95
+ # self.est_dir = args['est_dir']
96
+ # self.trn_proj = args['trn_proj']
97
+ # self.est_proj = args['est_proj']
98
+ # self.est_prefix = args['est_prefix']
99
+ # self.batch_size = args['trn_batch_size']
100
+ # self.num_char = args['num_char']
101
+ # self.num_states = args['num_states']
102
+ # self.num_epochs = args['num_epochs']
103
+ # self.tree_width = args['tree_width']
104
+ # self.tree_encode = args['tree_encode']
105
+ # self.char_encode = args['char_encode']
106
+ # self.brlen_encode = args['brlen_encode']
107
+ return
108
+
109
+ def prepare_filepaths(self):
110
+ """
111
+ Prepare filepaths for the project.
112
+
113
+ This script generates all the filepaths for input and output based off
114
+ of Trainer attributes. The Format and Train directories are input and
115
+ the Estimate directory is used for both input and output.
116
+
117
+ Returns: None
118
+ """
119
+ # main directories
120
+ self.trn_proj_dir = f'{self.trn_dir}/{self.trn_proj}'
121
+ self.est_proj_dir = f'{self.est_dir}/{self.est_proj}'
122
+ self.fmt_proj_dir = f'{self.fmt_dir}/{self.fmt_proj}'
123
+
124
+ # prefixes
125
+ test_prefix = f'test.nt{self.tree_width}'
126
+ network_prefix = f'network_nt{self.tree_width}'
127
+ #network_prefix = f'train_batchsize{self.trn_batch_size}_numepoch{self.num_epochs}_nt{self.tree_width}'
128
+ self.trn_prefix_dir = f'{self.trn_proj_dir}/{network_prefix}'
129
+ self.est_prefix_dir = f'{self.est_proj_dir}/{self.est_prefix}'
130
+ self.fmt_prefix_dir = f'{self.fmt_proj_dir}/{test_prefix}'
131
+
132
+ # model files
133
+ self.model_sav_fn = f'{self.trn_prefix_dir}.hdf5'
134
+ self.train_labels_norm_fn = f'{self.trn_prefix_dir}.train_label_norm.csv'
135
+ self.train_aux_data_norm_fn = f'{self.trn_prefix_dir}.train_aux_data_norm.csv'
136
+ self.model_cpi_fn = f'{self.trn_prefix_dir}.cpi_adjustments.csv'
137
+
138
+ # simulated test datasets for csv or hdf5
139
+ self.test_phy_data_fn = f'{self.fmt_prefix_dir}.phy_data.csv'
140
+ self.test_aux_data_fn = f'{self.fmt_prefix_dir}.aux_data.csv'
141
+ self.test_labels_fn = f'{self.fmt_prefix_dir}.labels.csv'
142
+ self.test_hdf5_fn = f'{self.fmt_prefix_dir}.hdf5'
143
+
144
+ # empirical test dataset
145
+ self.est_summ_stat_fn = f'{self.est_prefix_dir}.summ_stat.csv'
146
+ self.est_known_param_fn = f'{self.est_prefix_dir}.known_param.csv'
147
+ self.est_aux_data_fn = f'{self.est_prefix_dir}.aux_data.csv'
148
+ self.est_phy_data_fn = f'{self.est_prefix_dir}.phy_data.csv'
149
+
150
+ # test outputs
151
+ self.out_emp_label_est_fn = f'{self.est_prefix_dir}.emp_est.labels.csv'
152
+ self.out_test_label_est_fn = f'{self.est_prefix_dir}.test_est.labels.csv'
153
+ self.out_test_label_true_fn = f'{self.est_prefix_dir}.test_true.labels.csv'
154
+
155
+ # check if empirical dataset exists
156
+ self.emp_input_exists = True
157
+ for fn in [ self.est_summ_stat_fn,
158
+ self.est_phy_data_fn ]:
159
+ if not os.path.exists(fn):
160
+ self.emp_input_exists = False
161
+
162
+ # done
163
+ return
164
+
165
+ def run(self):
166
+ """
167
+ Executes all simulations.
168
+
169
+ This method prints status updates, creates the target directory for new
170
+ simulations, then runs all simulation jobs.
171
+
172
+ Simulation jobs are numbered by the replicate-index list (self.rep_idx).
173
+ Each job is executed by calling self.sim_one(idx) where idx is a unique
174
+ value in self.rep_idx.
175
+
176
+ When self.use_parallel is True then all jobs are run in parallel via
177
+ multiprocessing.Pool. When self.use_parallel is false, jobs are run
178
+ serially with one CPU.
179
+ """
180
+ verbose = self.verbose
181
+
182
+ # print header
183
+ util.print_step_header('est', [self.est_proj_dir, self.trn_proj_dir],
184
+ self.est_proj_dir, verbose)
185
+
186
+ # prepare workspace
187
+ os.makedirs(self.est_proj_dir, exist_ok=True)
188
+
189
+ # start time
190
+ start_time,start_time_str = util.get_time()
191
+ util.print_str(f'▪ Start time of {start_time_str}', verbose)
192
+
193
+ # load input
194
+ util.print_str('▪ Loading input', verbose)
195
+ self.load_input()
196
+
197
+ # make estimates
198
+ util.print_str('▪ Making estimates', verbose)
199
+ self.make_results()
200
+
201
+ # end time
202
+ end_time,end_time_str = util.get_time()
203
+ run_time = util.get_time_diff(start_time, end_time)
204
+ # util.print_str(f'▪ End time: {end_time_str}', verbose)
205
+ util.print_str(f'▪ End time of {end_time_str} (+{run_time})', verbose)
206
+
207
+ # done
208
+ util.print_str('... done!', verbose)
209
+
210
+ def load_input(self):
211
+ """
212
+ Load input data for estimation.
213
+
214
+ This function loads input from Train and Estimate. From Train, it
215
+ imports the trained network, scaling factors for the aux. data and
216
+ labels. It also loads the phy. data and aux. data tensors stored in the
217
+ Estimate job directory.
218
+
219
+ The script re-normalizes the new estimation to match the scale/location
220
+ used for simulated training examples to train the network.
221
+ """
222
+
223
+
224
+ # read & reshape new phylo-state data
225
+ self.emp_phy_data = None
226
+ if self.emp_input_exists:
227
+ self.emp_phy_data = pd.read_csv(self.est_phy_data_fn,
228
+ header=None, sep=',',
229
+ index_col=False).to_numpy()
230
+ self.emp_phy_data = self.emp_phy_data.reshape((1, -1, self.num_data_row))
231
+
232
+ # denormalization factors for new aux data
233
+ train_aux_data_norm = pd.read_csv(self.train_aux_data_norm_fn, sep=',', index_col=False)
234
+ train_aux_data_means = train_aux_data_norm['mean'].T.to_numpy().flatten()
235
+ train_aux_data_sd = train_aux_data_norm['sd'].T.to_numpy().flatten()
236
+ self.train_aux_data_mean_sd = (train_aux_data_means, train_aux_data_sd)
237
+
238
+ # denormalization factors for labels
239
+ train_labels_norm = pd.read_csv(self.train_labels_norm_fn, sep=',', index_col=False)
240
+ train_labels_means = train_labels_norm['mean'].T.to_numpy().flatten()
241
+ train_labels_sd = train_labels_norm['sd'].T.to_numpy().flatten()
242
+ self.train_labels_mean_sd = (train_labels_means, train_labels_sd)
243
+
244
+ # get param_names from training labels
245
+ self.aux_data_names = train_aux_data_norm['name'].to_list()
246
+ self.label_names = train_labels_norm['name'].to_list()
247
+
248
+ # read & normalize new aux data (when files exist)
249
+ self.emp_aux_data = None
250
+
251
+ if self.emp_input_exists:
252
+ self.est_summ_stat = pd.read_csv(self.est_summ_stat_fn, sep=',',
253
+ index_col=False).to_numpy().flatten()
254
+ try:
255
+ self.emp_known_params = pd.read_csv(self.est_known_param_fn,
256
+ sep=',', index_col=False).to_numpy().flatten()
257
+ self.emp_aux_data = np.concatenate([self.est_summ_stat,
258
+ self.emp_known_params])
259
+ except FileNotFoundError:
260
+ self.emp_aux_data = self.est_summ_stat
261
+
262
+ # reshape and rescale new aux data
263
+ self.emp_aux_data.shape = (1, -1)
264
+ self.norm_emp_aux_data = util.normalize(self.emp_aux_data,
265
+ self.train_aux_data_mean_sd)
266
+
267
+ # read in CQR interval adjustments
268
+ self.cpi_adjustments = pd.read_csv(self.model_cpi_fn, sep=',', index_col=False).to_numpy()
269
+
270
+
271
+ # TEST DATA
272
+ # load all the test dataset
273
+ if self.tensor_format == 'csv':
274
+ test_phy_data = pd.read_csv(self.test_phy_data_fn, header=None,
275
+ on_bad_lines='skip').to_numpy()
276
+ test_aux_data = pd.read_csv(self.test_aux_data_fn, header=None,
277
+ on_bad_lines='skip').to_numpy()
278
+ test_labels = pd.read_csv(self.test_labels_fn, header=None,
279
+ on_bad_lines='skip').to_numpy()
280
+ # self.aux_data_names = test_aux_data[0,:]
281
+ # self.label_names = test_labels[0,:]
282
+ test_aux_data = test_aux_data[1:,:].astype('float64')
283
+ test_labels = test_labels[1:,:].astype('float64')
284
+
285
+ elif self.tensor_format == 'hdf5':
286
+ hdf5_file = h5py.File(self.test_hdf5_fn, 'r')
287
+ # self.aux_data_names = [ s.decode() for s in hdf5_file['aux_data_names'][0,:] ]
288
+ # self.label_names = [ s.decode() for s in hdf5_file['label_names'][0,:] ]
289
+ test_phy_data = pd.DataFrame(hdf5_file['phy_data']).to_numpy()
290
+ test_aux_data = pd.DataFrame(hdf5_file['aux_data']).to_numpy()
291
+ test_labels = pd.DataFrame(hdf5_file['labels']).to_numpy()
292
+ hdf5_file.close()
293
+
294
+ num_sample = test_phy_data.shape[0]
295
+ test_phy_data.shape = (num_sample, -1, self.num_data_row)
296
+
297
+ # create phylogenetic data tensors
298
+ self.test_phy_data = test_phy_data
299
+ # self.test_aux_data = test_aux_data
300
+ self.norm_test_aux_data = util.normalize(test_aux_data,
301
+ self.train_aux_data_mean_sd)
302
+ # self.norm_test_labels = util.normalize(test_labels,
303
+ # self.train_labels_mean_sd)
304
+ self.test_label_true = test_labels
305
+
306
+ return
307
+
308
+
309
+ def make_results(self):
310
+ """
311
+ Makes all results for the Estimate step.
312
+
313
+ This function loads a trained model from the Train stem, then uses it
314
+ to perform the estimation task. For example, the step might estimate all
315
+ model parameter values and adjusted lower and upper CPI bounds. This step
316
+ """
317
+
318
+ # load model
319
+ self.mymodel = tf.keras.models.load_model(self.model_sav_fn, compile=False)
320
+
321
+ # empirical dataset (if it exists)
322
+ if self.emp_input_exists:
323
+
324
+ # get estimates
325
+ norm_emp_label_est = self.mymodel.predict([self.emp_phy_data,
326
+ self.norm_emp_aux_data])
327
+
328
+ # point estimates & CPIs for emp. labels
329
+ norm_emp_label_est = np.array(norm_emp_label_est)
330
+ norm_emp_label_est[1,:,:] = norm_emp_label_est[1,:,:] - self.cpi_adjustments[0,:]
331
+ norm_emp_label_est[2,:,:] = norm_emp_label_est[2,:,:] + self.cpi_adjustments[1,:]
332
+
333
+ # detransform results
334
+ # log_emp_label_est = util.denormalize(norm_emp_label_est,
335
+ # self.train_labels_mean_sd[0],
336
+ # self.train_labels_mean_sd[1])
337
+ # log_emp_label_est[log_emp_label_est > 300.] = 300.
338
+ # emp_label_est = np.exp(log_emp_label_est)
339
+ emp_label_est = util.denormalize(norm_emp_label_est,
340
+ self.train_labels_mean_sd,
341
+ exp=True)
342
+
343
+ # save empirial label estimates
344
+ df_emp_label_est = util.make_param_VLU_mtx(emp_label_est, self.label_names)
345
+ df_emp_label_est.to_csv(self.out_emp_label_est_fn, index=False, sep=',')
346
+
347
+ # save empirical auxiliary dataset
348
+ df_emp_aux_data = pd.DataFrame(self.emp_aux_data, columns=self.aux_data_names)
349
+ df_emp_aux_data.to_csv(self.est_aux_data_fn, index=False, sep=',')
350
+
351
+
352
+ # test dataset
353
+ norm_test_label_est = self.mymodel.predict([self.test_phy_data,
354
+ self.norm_test_aux_data])
355
+
356
+ # point estimates & CPIs for test labels
357
+ norm_test_label_est = np.array(norm_test_label_est)
358
+ norm_test_label_est[1,:,:] = norm_test_label_est[1,:,:] - self.cpi_adjustments[0,:]
359
+ norm_test_label_est[2,:,:] = norm_test_label_est[2,:,:] + self.cpi_adjustments[1,:]
360
+
361
+ # detransform test label estimates
362
+ # log_test_label_est = util.denormalize(norm_test_label_est,
363
+ # self.train_labels_mean_sd[0],
364
+ # self.train_labels_mean_sd[1])
365
+ # log_test_label_est[log_test_label_est > 300.] = 300.
366
+ # test_label_est = np.exp(log_test_label_est)
367
+ test_label_est = util.denormalize(norm_test_label_est,
368
+ self.train_labels_mean_sd,
369
+ exp=True)
370
+
371
+ #save test estimates
372
+ df_test_label_est = util.make_param_VLU_mtx(test_label_est, self.label_names)
373
+ df_test_label_est.to_csv(self.out_test_label_est_fn, index=False, sep=',')
374
+ df_test_label_true = pd.DataFrame(self.test_label_true, columns=self.label_names)
375
+ df_test_label_true.to_csv(self.out_test_label_true_fn, index=False, sep=',')
376
+
377
+ # done
378
+ return
379
+
380
+ #------------------------------------------------------------------------------#
381
+
382
+
383
+ # # scatter of estimate vs true for test data
384
+ # self.normalized_test_ests = self.mymodel.predict([self.test_phy_data_tensor, self.test_sux_data_tensor])
385
+ # self.normalized_test_ests = np.array(self.normalized_test_ests)
386
+ # self.denormalized_test_ests = util.denormalize(self.normalized_test_ests, self.train_label_means, self.train_label_sd)
387
+ # self.denormalized_test_ests = np.exp(self.denormalized_test_ests)
388
+ # self.denormalized_test_labels = util.denormalize(self.norm_test_labels, self.train_label_means, self.train_label_sd)
389
+ # self.denormalized_test_labels = np.exp(self.denormalized_test_labels)
390
+
391
+ # # test predictions with calibrated CQR CIs
392
+ # self.denorm_test_ests_calib = self.normalized_test_ests
393
+ # self.denorm_test_ests_calib[1,:,:] = self.denorm_test_ests_calib[1,:,:] - self.cpi_adjustments[0,:]
394
+ # self.denorm_test_ests_calib[2,:,:] = self.denorm_test_ests_calib[2,:,:] + self.cpi_adjustments[1,:]
395
+ # self.denorm_test_ests_calib = util.denormalize(self.denorm_test_ests_calib, self.train_label_means, self.train_label_sd)
396
+ # self.denorm_test_ests_calib = np.exp(self.denorm_test_ests_calib)
397
+
398
+ # # test scatterplot results (Value, Lower, Upper)
399
+ # df_test_est_nocalib = util.make_param_VLU_mtx(self.denormalized_test_ests[0:max_idx,:], self.label_names )
400
+ # df_test_est_calib = util.make_param_VLU_mtx(self.denorm_test_ests_calib[0:max_idx,:], self.label_names )
401
+
402
+ # # save train/test labels
403
+ # df_test_labels = pd.DataFrame( self.denormalized_test_labels[0:max_idx,:], columns=self.label_names )
404
+
405
+ # # convert to csv and save
406
+ # df_test_est_nocalib.to_csv(self.test_est_nocalib_fn, index=False, sep=',')
407
+ # df_test_est_calib.to_csv(self.test_est_calib_fn, index=False, sep=',')
408
+ # df_test_labels.to_csv(self.test_labels_fn, index=False, sep=',')
409
+