phyddle 0.0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phyddle/__init__.py +33 -0
- phyddle/__main__.py +8 -0
- phyddle/command_line.py +53 -0
- phyddle/estimate.py +409 -0
- phyddle/format.py +1052 -0
- phyddle/pipeline_step.py +52 -0
- phyddle/plot.py +1021 -0
- phyddle/simulate.py +242 -0
- phyddle/train.py +777 -0
- phyddle/utilities.py +2011 -0
- phyddle-0.0.7.dist-info/METADATA +132 -0
- phyddle-0.0.7.dist-info/RECORD +15 -0
- phyddle-0.0.7.dist-info/WHEEL +4 -0
- phyddle-0.0.7.dist-info/entry_points.txt +2 -0
- phyddle-0.0.7.dist-info/licenses/LICENSE +21 -0
phyddle/__init__.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
"""
|
|
3
|
+
__init__
|
|
4
|
+
===========
|
|
5
|
+
Initializes phyddle package and discovers all modules.
|
|
6
|
+
|
|
7
|
+
Authors: Michael Landis, Ammon Thompson
|
|
8
|
+
Copyright: (c) 2022-2023, Michael Landis and Ammon Thompson
|
|
9
|
+
License: MIT
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
#import os
|
|
13
|
+
#import sys
|
|
14
|
+
from multiprocessing import set_start_method
|
|
15
|
+
|
|
16
|
+
__project__ = 'phyddle'
|
|
17
|
+
__version__ = '0.0.7'
|
|
18
|
+
__author__ = 'Michael Landis and Ammon Thompson'
|
|
19
|
+
__copyright__ = '(c) 2022-2023, Michael Landis and Ammon Thompson'
|
|
20
|
+
__citation__ = 'A Thompson, B Liebeskind, EJ Scully, MJ Landis. 2023. Deep learning approaches to viral phylogeography are fast and as robust as likelihood methods to model misspecification. bioRxiv 2023.02.08.527714.'
|
|
21
|
+
|
|
22
|
+
# DEFAULT
|
|
23
|
+
CONFIG_DEFAULT_FN = '__config_default.py'
|
|
24
|
+
PHYDDLE_VERSION = __version__
|
|
25
|
+
|
|
26
|
+
# add local directory to system path
|
|
27
|
+
# allows import of local cfg
|
|
28
|
+
# ...but might switch over to simple tab-deilmited file
|
|
29
|
+
#sys.path.insert(0, os.getcwd())
|
|
30
|
+
|
|
31
|
+
if __name__ == '__main__':
|
|
32
|
+
# set the start method
|
|
33
|
+
set_start_method('fork')
|
phyddle/__main__.py
ADDED
phyddle/command_line.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
def run():
|
|
4
|
+
|
|
5
|
+
########################
|
|
6
|
+
# LOAD PIPELINE CONFIG #
|
|
7
|
+
########################
|
|
8
|
+
|
|
9
|
+
import phyddle.utilities as util
|
|
10
|
+
my_args = util.load_config('config', arg_overwrite=True)
|
|
11
|
+
step = my_args['step']
|
|
12
|
+
|
|
13
|
+
logger = util.Logger(my_args)
|
|
14
|
+
logger.save_log('run')
|
|
15
|
+
|
|
16
|
+
################
|
|
17
|
+
# RUN PIPELINE #
|
|
18
|
+
################
|
|
19
|
+
|
|
20
|
+
# Step 1: simulate training data
|
|
21
|
+
if 'S' in step:
|
|
22
|
+
import phyddle.simulate as sim
|
|
23
|
+
my_sim = sim.load(my_args)
|
|
24
|
+
my_sim.run()
|
|
25
|
+
|
|
26
|
+
# Step 2: format training data into tensors
|
|
27
|
+
if 'F' in step:
|
|
28
|
+
import phyddle.format as fmt
|
|
29
|
+
my_fmt = fmt.load(my_args)
|
|
30
|
+
my_fmt.run()
|
|
31
|
+
|
|
32
|
+
# Step 3: train network with training data
|
|
33
|
+
if 'T' in step:
|
|
34
|
+
import phyddle.train as trn
|
|
35
|
+
my_trn = trn.load(my_args)
|
|
36
|
+
my_trn.run()
|
|
37
|
+
|
|
38
|
+
# Step 4: estimates for new dataset
|
|
39
|
+
if 'E' in step:
|
|
40
|
+
import phyddle.format as fmt
|
|
41
|
+
import phyddle.estimate as est
|
|
42
|
+
est_prefix = f"{my_args['est_dir']}/{my_args['proj']}/{my_args['est_prefix']}"
|
|
43
|
+
my_fmt = fmt.load(my_args)
|
|
44
|
+
my_fmt.encode_one(tmp_fn=est_prefix, idx=-1, save_phyenc_csv=True)
|
|
45
|
+
my_est = est.load(my_args)
|
|
46
|
+
my_est.run()
|
|
47
|
+
|
|
48
|
+
# Step 5: plot results
|
|
49
|
+
if 'P' in step:
|
|
50
|
+
import phyddle.plot as plt
|
|
51
|
+
my_plt = plt.load(my_args)
|
|
52
|
+
my_plt.run()
|
|
53
|
+
|
phyddle/estimate.py
ADDED
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
"""
|
|
3
|
+
estimate
|
|
4
|
+
========
|
|
5
|
+
Defines classes and methods for the Estimate step, which loads a pre-trained
|
|
6
|
+
network and uses it to generate new estimates, e.g. estimate model parmaeters
|
|
7
|
+
for a new empirical dataset.
|
|
8
|
+
|
|
9
|
+
Authors: Michael Landis and Ammon Thompson
|
|
10
|
+
Copyright: (c) 2022-2023, Michael Landis and Ammon Thompson
|
|
11
|
+
License: MIT
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
# standard imports
|
|
15
|
+
import os
|
|
16
|
+
|
|
17
|
+
# external imports
|
|
18
|
+
import numpy as np
|
|
19
|
+
import pandas as pd
|
|
20
|
+
import tensorflow as tf
|
|
21
|
+
import h5py
|
|
22
|
+
|
|
23
|
+
# phyddle imports
|
|
24
|
+
from phyddle import utilities as util
|
|
25
|
+
|
|
26
|
+
#------------------------------------------------------------------------------#
|
|
27
|
+
|
|
28
|
+
def load(args):
|
|
29
|
+
"""
|
|
30
|
+
Load an Estimator object.
|
|
31
|
+
|
|
32
|
+
This function creates an instance of the Estimator class, initialized using
|
|
33
|
+
phyddle settings stored in args (dict).
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
args (dict): Contains phyddle settings.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
# load object
|
|
40
|
+
est_method = 'default'
|
|
41
|
+
if est_method == 'default':
|
|
42
|
+
return Estimator(args)
|
|
43
|
+
else:
|
|
44
|
+
return NotImplementedError
|
|
45
|
+
|
|
46
|
+
#------------------------------------------------------------------------------#
|
|
47
|
+
|
|
48
|
+
class Estimator:
|
|
49
|
+
"""
|
|
50
|
+
Class for making neural network estimates (i.e. label predictions) from new
|
|
51
|
+
(e.g. empirical) phylogenetic datasets. This class requires a trained
|
|
52
|
+
network from Train and input processed by Format. Output is written to
|
|
53
|
+
file and can be visualized using Plot.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
def __init__(self, args):
|
|
57
|
+
"""
|
|
58
|
+
Initializes a new Simulator object.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
args (dict): Contains phyddle settings.
|
|
62
|
+
"""
|
|
63
|
+
# initialize with phyddle settings
|
|
64
|
+
self.set_args(args)
|
|
65
|
+
# construct filepaths
|
|
66
|
+
self.prepare_filepaths()
|
|
67
|
+
# get size of CPV+S tensors
|
|
68
|
+
self.num_tree_row = util.get_num_tree_row(self.tree_encode,
|
|
69
|
+
self.brlen_encode)
|
|
70
|
+
self.num_char_row = util.get_num_char_row(self.char_encode,
|
|
71
|
+
self.num_char,
|
|
72
|
+
self.num_states)
|
|
73
|
+
self.num_data_row = self.num_tree_row + self.num_char_row
|
|
74
|
+
# create logger to track runtime info
|
|
75
|
+
self.logger = util.Logger(args)
|
|
76
|
+
# done
|
|
77
|
+
return
|
|
78
|
+
|
|
79
|
+
def set_args(self, args):
|
|
80
|
+
"""
|
|
81
|
+
Assigns phyddle settings as Estimator attributes.
|
|
82
|
+
|
|
83
|
+
Args:
|
|
84
|
+
args (dict): Contains phyddle settings.
|
|
85
|
+
"""
|
|
86
|
+
# estimator arguments
|
|
87
|
+
self.args = args
|
|
88
|
+
step_args = util.make_step_args('E', args)
|
|
89
|
+
for k,v in step_args.items():
|
|
90
|
+
setattr(self, k, v)
|
|
91
|
+
|
|
92
|
+
# self.args = args
|
|
93
|
+
# self.verbose = args['verbose']
|
|
94
|
+
# self.trn_dir = args['trn_dir']
|
|
95
|
+
# self.est_dir = args['est_dir']
|
|
96
|
+
# self.trn_proj = args['trn_proj']
|
|
97
|
+
# self.est_proj = args['est_proj']
|
|
98
|
+
# self.est_prefix = args['est_prefix']
|
|
99
|
+
# self.batch_size = args['trn_batch_size']
|
|
100
|
+
# self.num_char = args['num_char']
|
|
101
|
+
# self.num_states = args['num_states']
|
|
102
|
+
# self.num_epochs = args['num_epochs']
|
|
103
|
+
# self.tree_width = args['tree_width']
|
|
104
|
+
# self.tree_encode = args['tree_encode']
|
|
105
|
+
# self.char_encode = args['char_encode']
|
|
106
|
+
# self.brlen_encode = args['brlen_encode']
|
|
107
|
+
return
|
|
108
|
+
|
|
109
|
+
def prepare_filepaths(self):
|
|
110
|
+
"""
|
|
111
|
+
Prepare filepaths for the project.
|
|
112
|
+
|
|
113
|
+
This script generates all the filepaths for input and output based off
|
|
114
|
+
of Trainer attributes. The Format and Train directories are input and
|
|
115
|
+
the Estimate directory is used for both input and output.
|
|
116
|
+
|
|
117
|
+
Returns: None
|
|
118
|
+
"""
|
|
119
|
+
# main directories
|
|
120
|
+
self.trn_proj_dir = f'{self.trn_dir}/{self.trn_proj}'
|
|
121
|
+
self.est_proj_dir = f'{self.est_dir}/{self.est_proj}'
|
|
122
|
+
self.fmt_proj_dir = f'{self.fmt_dir}/{self.fmt_proj}'
|
|
123
|
+
|
|
124
|
+
# prefixes
|
|
125
|
+
test_prefix = f'test.nt{self.tree_width}'
|
|
126
|
+
network_prefix = f'network_nt{self.tree_width}'
|
|
127
|
+
#network_prefix = f'train_batchsize{self.trn_batch_size}_numepoch{self.num_epochs}_nt{self.tree_width}'
|
|
128
|
+
self.trn_prefix_dir = f'{self.trn_proj_dir}/{network_prefix}'
|
|
129
|
+
self.est_prefix_dir = f'{self.est_proj_dir}/{self.est_prefix}'
|
|
130
|
+
self.fmt_prefix_dir = f'{self.fmt_proj_dir}/{test_prefix}'
|
|
131
|
+
|
|
132
|
+
# model files
|
|
133
|
+
self.model_sav_fn = f'{self.trn_prefix_dir}.hdf5'
|
|
134
|
+
self.train_labels_norm_fn = f'{self.trn_prefix_dir}.train_label_norm.csv'
|
|
135
|
+
self.train_aux_data_norm_fn = f'{self.trn_prefix_dir}.train_aux_data_norm.csv'
|
|
136
|
+
self.model_cpi_fn = f'{self.trn_prefix_dir}.cpi_adjustments.csv'
|
|
137
|
+
|
|
138
|
+
# simulated test datasets for csv or hdf5
|
|
139
|
+
self.test_phy_data_fn = f'{self.fmt_prefix_dir}.phy_data.csv'
|
|
140
|
+
self.test_aux_data_fn = f'{self.fmt_prefix_dir}.aux_data.csv'
|
|
141
|
+
self.test_labels_fn = f'{self.fmt_prefix_dir}.labels.csv'
|
|
142
|
+
self.test_hdf5_fn = f'{self.fmt_prefix_dir}.hdf5'
|
|
143
|
+
|
|
144
|
+
# empirical test dataset
|
|
145
|
+
self.est_summ_stat_fn = f'{self.est_prefix_dir}.summ_stat.csv'
|
|
146
|
+
self.est_known_param_fn = f'{self.est_prefix_dir}.known_param.csv'
|
|
147
|
+
self.est_aux_data_fn = f'{self.est_prefix_dir}.aux_data.csv'
|
|
148
|
+
self.est_phy_data_fn = f'{self.est_prefix_dir}.phy_data.csv'
|
|
149
|
+
|
|
150
|
+
# test outputs
|
|
151
|
+
self.out_emp_label_est_fn = f'{self.est_prefix_dir}.emp_est.labels.csv'
|
|
152
|
+
self.out_test_label_est_fn = f'{self.est_prefix_dir}.test_est.labels.csv'
|
|
153
|
+
self.out_test_label_true_fn = f'{self.est_prefix_dir}.test_true.labels.csv'
|
|
154
|
+
|
|
155
|
+
# check if empirical dataset exists
|
|
156
|
+
self.emp_input_exists = True
|
|
157
|
+
for fn in [ self.est_summ_stat_fn,
|
|
158
|
+
self.est_phy_data_fn ]:
|
|
159
|
+
if not os.path.exists(fn):
|
|
160
|
+
self.emp_input_exists = False
|
|
161
|
+
|
|
162
|
+
# done
|
|
163
|
+
return
|
|
164
|
+
|
|
165
|
+
def run(self):
|
|
166
|
+
"""
|
|
167
|
+
Executes all simulations.
|
|
168
|
+
|
|
169
|
+
This method prints status updates, creates the target directory for new
|
|
170
|
+
simulations, then runs all simulation jobs.
|
|
171
|
+
|
|
172
|
+
Simulation jobs are numbered by the replicate-index list (self.rep_idx).
|
|
173
|
+
Each job is executed by calling self.sim_one(idx) where idx is a unique
|
|
174
|
+
value in self.rep_idx.
|
|
175
|
+
|
|
176
|
+
When self.use_parallel is True then all jobs are run in parallel via
|
|
177
|
+
multiprocessing.Pool. When self.use_parallel is false, jobs are run
|
|
178
|
+
serially with one CPU.
|
|
179
|
+
"""
|
|
180
|
+
verbose = self.verbose
|
|
181
|
+
|
|
182
|
+
# print header
|
|
183
|
+
util.print_step_header('est', [self.est_proj_dir, self.trn_proj_dir],
|
|
184
|
+
self.est_proj_dir, verbose)
|
|
185
|
+
|
|
186
|
+
# prepare workspace
|
|
187
|
+
os.makedirs(self.est_proj_dir, exist_ok=True)
|
|
188
|
+
|
|
189
|
+
# start time
|
|
190
|
+
start_time,start_time_str = util.get_time()
|
|
191
|
+
util.print_str(f'▪ Start time of {start_time_str}', verbose)
|
|
192
|
+
|
|
193
|
+
# load input
|
|
194
|
+
util.print_str('▪ Loading input', verbose)
|
|
195
|
+
self.load_input()
|
|
196
|
+
|
|
197
|
+
# make estimates
|
|
198
|
+
util.print_str('▪ Making estimates', verbose)
|
|
199
|
+
self.make_results()
|
|
200
|
+
|
|
201
|
+
# end time
|
|
202
|
+
end_time,end_time_str = util.get_time()
|
|
203
|
+
run_time = util.get_time_diff(start_time, end_time)
|
|
204
|
+
# util.print_str(f'▪ End time: {end_time_str}', verbose)
|
|
205
|
+
util.print_str(f'▪ End time of {end_time_str} (+{run_time})', verbose)
|
|
206
|
+
|
|
207
|
+
# done
|
|
208
|
+
util.print_str('... done!', verbose)
|
|
209
|
+
|
|
210
|
+
def load_input(self):
|
|
211
|
+
"""
|
|
212
|
+
Load input data for estimation.
|
|
213
|
+
|
|
214
|
+
This function loads input from Train and Estimate. From Train, it
|
|
215
|
+
imports the trained network, scaling factors for the aux. data and
|
|
216
|
+
labels. It also loads the phy. data and aux. data tensors stored in the
|
|
217
|
+
Estimate job directory.
|
|
218
|
+
|
|
219
|
+
The script re-normalizes the new estimation to match the scale/location
|
|
220
|
+
used for simulated training examples to train the network.
|
|
221
|
+
"""
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
# read & reshape new phylo-state data
|
|
225
|
+
self.emp_phy_data = None
|
|
226
|
+
if self.emp_input_exists:
|
|
227
|
+
self.emp_phy_data = pd.read_csv(self.est_phy_data_fn,
|
|
228
|
+
header=None, sep=',',
|
|
229
|
+
index_col=False).to_numpy()
|
|
230
|
+
self.emp_phy_data = self.emp_phy_data.reshape((1, -1, self.num_data_row))
|
|
231
|
+
|
|
232
|
+
# denormalization factors for new aux data
|
|
233
|
+
train_aux_data_norm = pd.read_csv(self.train_aux_data_norm_fn, sep=',', index_col=False)
|
|
234
|
+
train_aux_data_means = train_aux_data_norm['mean'].T.to_numpy().flatten()
|
|
235
|
+
train_aux_data_sd = train_aux_data_norm['sd'].T.to_numpy().flatten()
|
|
236
|
+
self.train_aux_data_mean_sd = (train_aux_data_means, train_aux_data_sd)
|
|
237
|
+
|
|
238
|
+
# denormalization factors for labels
|
|
239
|
+
train_labels_norm = pd.read_csv(self.train_labels_norm_fn, sep=',', index_col=False)
|
|
240
|
+
train_labels_means = train_labels_norm['mean'].T.to_numpy().flatten()
|
|
241
|
+
train_labels_sd = train_labels_norm['sd'].T.to_numpy().flatten()
|
|
242
|
+
self.train_labels_mean_sd = (train_labels_means, train_labels_sd)
|
|
243
|
+
|
|
244
|
+
# get param_names from training labels
|
|
245
|
+
self.aux_data_names = train_aux_data_norm['name'].to_list()
|
|
246
|
+
self.label_names = train_labels_norm['name'].to_list()
|
|
247
|
+
|
|
248
|
+
# read & normalize new aux data (when files exist)
|
|
249
|
+
self.emp_aux_data = None
|
|
250
|
+
|
|
251
|
+
if self.emp_input_exists:
|
|
252
|
+
self.est_summ_stat = pd.read_csv(self.est_summ_stat_fn, sep=',',
|
|
253
|
+
index_col=False).to_numpy().flatten()
|
|
254
|
+
try:
|
|
255
|
+
self.emp_known_params = pd.read_csv(self.est_known_param_fn,
|
|
256
|
+
sep=',', index_col=False).to_numpy().flatten()
|
|
257
|
+
self.emp_aux_data = np.concatenate([self.est_summ_stat,
|
|
258
|
+
self.emp_known_params])
|
|
259
|
+
except FileNotFoundError:
|
|
260
|
+
self.emp_aux_data = self.est_summ_stat
|
|
261
|
+
|
|
262
|
+
# reshape and rescale new aux data
|
|
263
|
+
self.emp_aux_data.shape = (1, -1)
|
|
264
|
+
self.norm_emp_aux_data = util.normalize(self.emp_aux_data,
|
|
265
|
+
self.train_aux_data_mean_sd)
|
|
266
|
+
|
|
267
|
+
# read in CQR interval adjustments
|
|
268
|
+
self.cpi_adjustments = pd.read_csv(self.model_cpi_fn, sep=',', index_col=False).to_numpy()
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
# TEST DATA
|
|
272
|
+
# load all the test dataset
|
|
273
|
+
if self.tensor_format == 'csv':
|
|
274
|
+
test_phy_data = pd.read_csv(self.test_phy_data_fn, header=None,
|
|
275
|
+
on_bad_lines='skip').to_numpy()
|
|
276
|
+
test_aux_data = pd.read_csv(self.test_aux_data_fn, header=None,
|
|
277
|
+
on_bad_lines='skip').to_numpy()
|
|
278
|
+
test_labels = pd.read_csv(self.test_labels_fn, header=None,
|
|
279
|
+
on_bad_lines='skip').to_numpy()
|
|
280
|
+
# self.aux_data_names = test_aux_data[0,:]
|
|
281
|
+
# self.label_names = test_labels[0,:]
|
|
282
|
+
test_aux_data = test_aux_data[1:,:].astype('float64')
|
|
283
|
+
test_labels = test_labels[1:,:].astype('float64')
|
|
284
|
+
|
|
285
|
+
elif self.tensor_format == 'hdf5':
|
|
286
|
+
hdf5_file = h5py.File(self.test_hdf5_fn, 'r')
|
|
287
|
+
# self.aux_data_names = [ s.decode() for s in hdf5_file['aux_data_names'][0,:] ]
|
|
288
|
+
# self.label_names = [ s.decode() for s in hdf5_file['label_names'][0,:] ]
|
|
289
|
+
test_phy_data = pd.DataFrame(hdf5_file['phy_data']).to_numpy()
|
|
290
|
+
test_aux_data = pd.DataFrame(hdf5_file['aux_data']).to_numpy()
|
|
291
|
+
test_labels = pd.DataFrame(hdf5_file['labels']).to_numpy()
|
|
292
|
+
hdf5_file.close()
|
|
293
|
+
|
|
294
|
+
num_sample = test_phy_data.shape[0]
|
|
295
|
+
test_phy_data.shape = (num_sample, -1, self.num_data_row)
|
|
296
|
+
|
|
297
|
+
# create phylogenetic data tensors
|
|
298
|
+
self.test_phy_data = test_phy_data
|
|
299
|
+
# self.test_aux_data = test_aux_data
|
|
300
|
+
self.norm_test_aux_data = util.normalize(test_aux_data,
|
|
301
|
+
self.train_aux_data_mean_sd)
|
|
302
|
+
# self.norm_test_labels = util.normalize(test_labels,
|
|
303
|
+
# self.train_labels_mean_sd)
|
|
304
|
+
self.test_label_true = test_labels
|
|
305
|
+
|
|
306
|
+
return
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def make_results(self):
|
|
310
|
+
"""
|
|
311
|
+
Makes all results for the Estimate step.
|
|
312
|
+
|
|
313
|
+
This function loads a trained model from the Train stem, then uses it
|
|
314
|
+
to perform the estimation task. For example, the step might estimate all
|
|
315
|
+
model parameter values and adjusted lower and upper CPI bounds. This step
|
|
316
|
+
"""
|
|
317
|
+
|
|
318
|
+
# load model
|
|
319
|
+
self.mymodel = tf.keras.models.load_model(self.model_sav_fn, compile=False)
|
|
320
|
+
|
|
321
|
+
# empirical dataset (if it exists)
|
|
322
|
+
if self.emp_input_exists:
|
|
323
|
+
|
|
324
|
+
# get estimates
|
|
325
|
+
norm_emp_label_est = self.mymodel.predict([self.emp_phy_data,
|
|
326
|
+
self.norm_emp_aux_data])
|
|
327
|
+
|
|
328
|
+
# point estimates & CPIs for emp. labels
|
|
329
|
+
norm_emp_label_est = np.array(norm_emp_label_est)
|
|
330
|
+
norm_emp_label_est[1,:,:] = norm_emp_label_est[1,:,:] - self.cpi_adjustments[0,:]
|
|
331
|
+
norm_emp_label_est[2,:,:] = norm_emp_label_est[2,:,:] + self.cpi_adjustments[1,:]
|
|
332
|
+
|
|
333
|
+
# detransform results
|
|
334
|
+
# log_emp_label_est = util.denormalize(norm_emp_label_est,
|
|
335
|
+
# self.train_labels_mean_sd[0],
|
|
336
|
+
# self.train_labels_mean_sd[1])
|
|
337
|
+
# log_emp_label_est[log_emp_label_est > 300.] = 300.
|
|
338
|
+
# emp_label_est = np.exp(log_emp_label_est)
|
|
339
|
+
emp_label_est = util.denormalize(norm_emp_label_est,
|
|
340
|
+
self.train_labels_mean_sd,
|
|
341
|
+
exp=True)
|
|
342
|
+
|
|
343
|
+
# save empirial label estimates
|
|
344
|
+
df_emp_label_est = util.make_param_VLU_mtx(emp_label_est, self.label_names)
|
|
345
|
+
df_emp_label_est.to_csv(self.out_emp_label_est_fn, index=False, sep=',')
|
|
346
|
+
|
|
347
|
+
# save empirical auxiliary dataset
|
|
348
|
+
df_emp_aux_data = pd.DataFrame(self.emp_aux_data, columns=self.aux_data_names)
|
|
349
|
+
df_emp_aux_data.to_csv(self.est_aux_data_fn, index=False, sep=',')
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
# test dataset
|
|
353
|
+
norm_test_label_est = self.mymodel.predict([self.test_phy_data,
|
|
354
|
+
self.norm_test_aux_data])
|
|
355
|
+
|
|
356
|
+
# point estimates & CPIs for test labels
|
|
357
|
+
norm_test_label_est = np.array(norm_test_label_est)
|
|
358
|
+
norm_test_label_est[1,:,:] = norm_test_label_est[1,:,:] - self.cpi_adjustments[0,:]
|
|
359
|
+
norm_test_label_est[2,:,:] = norm_test_label_est[2,:,:] + self.cpi_adjustments[1,:]
|
|
360
|
+
|
|
361
|
+
# detransform test label estimates
|
|
362
|
+
# log_test_label_est = util.denormalize(norm_test_label_est,
|
|
363
|
+
# self.train_labels_mean_sd[0],
|
|
364
|
+
# self.train_labels_mean_sd[1])
|
|
365
|
+
# log_test_label_est[log_test_label_est > 300.] = 300.
|
|
366
|
+
# test_label_est = np.exp(log_test_label_est)
|
|
367
|
+
test_label_est = util.denormalize(norm_test_label_est,
|
|
368
|
+
self.train_labels_mean_sd,
|
|
369
|
+
exp=True)
|
|
370
|
+
|
|
371
|
+
#save test estimates
|
|
372
|
+
df_test_label_est = util.make_param_VLU_mtx(test_label_est, self.label_names)
|
|
373
|
+
df_test_label_est.to_csv(self.out_test_label_est_fn, index=False, sep=',')
|
|
374
|
+
df_test_label_true = pd.DataFrame(self.test_label_true, columns=self.label_names)
|
|
375
|
+
df_test_label_true.to_csv(self.out_test_label_true_fn, index=False, sep=',')
|
|
376
|
+
|
|
377
|
+
# done
|
|
378
|
+
return
|
|
379
|
+
|
|
380
|
+
#------------------------------------------------------------------------------#
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
# # scatter of estimate vs true for test data
|
|
384
|
+
# self.normalized_test_ests = self.mymodel.predict([self.test_phy_data_tensor, self.test_sux_data_tensor])
|
|
385
|
+
# self.normalized_test_ests = np.array(self.normalized_test_ests)
|
|
386
|
+
# self.denormalized_test_ests = util.denormalize(self.normalized_test_ests, self.train_label_means, self.train_label_sd)
|
|
387
|
+
# self.denormalized_test_ests = np.exp(self.denormalized_test_ests)
|
|
388
|
+
# self.denormalized_test_labels = util.denormalize(self.norm_test_labels, self.train_label_means, self.train_label_sd)
|
|
389
|
+
# self.denormalized_test_labels = np.exp(self.denormalized_test_labels)
|
|
390
|
+
|
|
391
|
+
# # test predictions with calibrated CQR CIs
|
|
392
|
+
# self.denorm_test_ests_calib = self.normalized_test_ests
|
|
393
|
+
# self.denorm_test_ests_calib[1,:,:] = self.denorm_test_ests_calib[1,:,:] - self.cpi_adjustments[0,:]
|
|
394
|
+
# self.denorm_test_ests_calib[2,:,:] = self.denorm_test_ests_calib[2,:,:] + self.cpi_adjustments[1,:]
|
|
395
|
+
# self.denorm_test_ests_calib = util.denormalize(self.denorm_test_ests_calib, self.train_label_means, self.train_label_sd)
|
|
396
|
+
# self.denorm_test_ests_calib = np.exp(self.denorm_test_ests_calib)
|
|
397
|
+
|
|
398
|
+
# # test scatterplot results (Value, Lower, Upper)
|
|
399
|
+
# df_test_est_nocalib = util.make_param_VLU_mtx(self.denormalized_test_ests[0:max_idx,:], self.label_names )
|
|
400
|
+
# df_test_est_calib = util.make_param_VLU_mtx(self.denorm_test_ests_calib[0:max_idx,:], self.label_names )
|
|
401
|
+
|
|
402
|
+
# # save train/test labels
|
|
403
|
+
# df_test_labels = pd.DataFrame( self.denormalized_test_labels[0:max_idx,:], columns=self.label_names )
|
|
404
|
+
|
|
405
|
+
# # convert to csv and save
|
|
406
|
+
# df_test_est_nocalib.to_csv(self.test_est_nocalib_fn, index=False, sep=',')
|
|
407
|
+
# df_test_est_calib.to_csv(self.test_est_calib_fn, index=False, sep=',')
|
|
408
|
+
# df_test_labels.to_csv(self.test_labels_fn, index=False, sep=',')
|
|
409
|
+
|