dietnetwork 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dietnet/__init__.py +11 -0
- dietnet/_launch.py +40 -0
- dietnet/_upstream/Dietnet_train_and_evaluate/__init__.py +0 -0
- dietnet/_upstream/Dietnet_train_and_evaluate/compute_input_features_statistics.py +221 -0
- dietnet/_upstream/Dietnet_train_and_evaluate/generate_embedding.py +216 -0
- dietnet/_upstream/Dietnet_train_and_evaluate/helpers/__init__.py +0 -0
- dietnet/_upstream/Dietnet_train_and_evaluate/helpers/dataset_utils.py +200 -0
- dietnet/_upstream/Dietnet_train_and_evaluate/helpers/log_utils.py +148 -0
- dietnet/_upstream/Dietnet_train_and_evaluate/helpers/mainloop_utils.py +395 -0
- dietnet/_upstream/Dietnet_train_and_evaluate/helpers/model.py +489 -0
- dietnet/_upstream/Dietnet_train_and_evaluate/helpers/model_handlers.py +89 -0
- dietnet/_upstream/Dietnet_train_and_evaluate/helpers/task_handlers.py +361 -0
- dietnet/_upstream/Dietnet_train_and_evaluate/partition_data.py +331 -0
- dietnet/_upstream/Dietnet_train_and_evaluate/train.py +959 -0
- dietnet/align.py +188 -0
- dietnet/alphagenome.py +208 -0
- dietnet/cli.py +479 -0
- dietnet/dataset.py +181 -0
- dietnet/features.py +267 -0
- dietnet/package.py +187 -0
- dietnet/plink.py +163 -0
- dietnet/predict.py +123 -0
- dietnet/published.py +87 -0
- dietnet/py.typed +0 -0
- dietnet/training.py +176 -0
- dietnet/upstream.py +161 -0
- dietnet/upstream_manifest.json +53 -0
- dietnetwork-0.3.0.dist-info/METADATA +249 -0
- dietnetwork-0.3.0.dist-info/RECORD +33 -0
- dietnetwork-0.3.0.dist-info/WHEEL +4 -0
- dietnetwork-0.3.0.dist-info/entry_points.txt +2 -0
- dietnetwork-0.3.0.dist-info/licenses/LICENSE +28 -0
- dietnetwork-0.3.0.dist-info/licenses/NOTICE +15 -0
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import inspect
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def create_dir(dir_fullpath):
|
|
12
|
+
"""
|
|
13
|
+
This check if a direcotry already exists and
|
|
14
|
+
creates the directory if it does not
|
|
15
|
+
"""
|
|
16
|
+
if not os.path.isdir(dir_fullpath):
|
|
17
|
+
try:
|
|
18
|
+
Path(dir_fullpath).mkdir(parents=True, exist_ok=True)
|
|
19
|
+
except OSError:
|
|
20
|
+
print('Creation of directory %s failed' % dir_fullpath)
|
|
21
|
+
sys.exit(1)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def save_exp_params(out_dir, filename, config):
|
|
25
|
+
"""
|
|
26
|
+
Creates the file exp_params.log where experiment
|
|
27
|
+
configurations will be saved for reproducibility
|
|
28
|
+
"""
|
|
29
|
+
with open(os.path.join(out_dir, filename), 'w') as f:
|
|
30
|
+
for k,v in config.items():
|
|
31
|
+
f.write(k + ':' + str(v) + '\n')
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def save_model_summary(out_dir, model, criterion, optimizer):
|
|
35
|
+
filename = 'model_summary.log'
|
|
36
|
+
|
|
37
|
+
print('Saving model architecture summary to %s' % os.path.join(
|
|
38
|
+
out_dir,
|
|
39
|
+
filename)
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
text = '*'*50
|
|
43
|
+
text += '\nAuxiliary network (feature embedding network):\n'
|
|
44
|
+
text += '*'*50 + '\n'
|
|
45
|
+
text += str(model.feat_emb)
|
|
46
|
+
text += '\nForward function of auxiliary net:\n'
|
|
47
|
+
text += inspect.getsource(model.feat_emb.forward)
|
|
48
|
+
text += '\n' + '-'*80
|
|
49
|
+
|
|
50
|
+
text += '\n' + '*'*40
|
|
51
|
+
text += '\nMain network (discriminative network):\n'
|
|
52
|
+
text += '*'*40 + '\n'
|
|
53
|
+
text += str(model.disc_net)
|
|
54
|
+
text += '\nForward function of main net:\n'
|
|
55
|
+
text += inspect.getsource(model.disc_net.forward)
|
|
56
|
+
bias = False if model.disc_net.fat_bias is None else True
|
|
57
|
+
text += '\n --> Fat layer has bias param:' + str(bias) + '\n'
|
|
58
|
+
text += '\n' + '-'*80
|
|
59
|
+
|
|
60
|
+
text += '\n' + '*'*20
|
|
61
|
+
text += '\nCombined models:\n'
|
|
62
|
+
text += '*'*20 + '\n'
|
|
63
|
+
text += str(model)
|
|
64
|
+
text += '\nForward function of combined models:\n'
|
|
65
|
+
text += inspect.getsource(model.disc_net.forward)
|
|
66
|
+
|
|
67
|
+
text += '\n' + '-'*80
|
|
68
|
+
text += '\n' + '*'*20
|
|
69
|
+
text += '\nLoss and Optimizer\n'
|
|
70
|
+
text += '*'*20 + '\n'
|
|
71
|
+
text += 'Loss:\n' + str(criterion)
|
|
72
|
+
text += '\n\nOptimizer:\n' + str(optimizer)
|
|
73
|
+
|
|
74
|
+
with open(os.path.join(out_dir, filename), 'w') as f:
|
|
75
|
+
f.write(text)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def save_model_params(out_dir, model, filename='model_params.pt'):
|
|
79
|
+
#filename = 'model_params.pt'
|
|
80
|
+
|
|
81
|
+
print('Saving model parameters to %s' % os.path.join(out_dir, filename))
|
|
82
|
+
|
|
83
|
+
torch.save(model.state_dict(), os.path.join(out_dir, filename))
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def save_results(out_dir, samples, labels, label_names, score, pred):
|
|
87
|
+
filename = 'model_predictions.npz'
|
|
88
|
+
|
|
89
|
+
print('Saving model predictions to %s' % os.path.join(out_dir, filename))
|
|
90
|
+
|
|
91
|
+
np.savez(os.path.join(out_dir, filename),
|
|
92
|
+
test_samples=samples,
|
|
93
|
+
test_labels=labels,
|
|
94
|
+
test_scores=score,
|
|
95
|
+
test_preds=pred,
|
|
96
|
+
label_names=label_names)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def save_results_regression(out_dir, samples, labels, pred):
|
|
100
|
+
filename = 'model_predictions.npz'
|
|
101
|
+
|
|
102
|
+
print('Saving model predictions to %s' % os.path.join(out_dir, filename))
|
|
103
|
+
|
|
104
|
+
np.savez(os.path.join(out_dir, filename),
|
|
105
|
+
test_samples=samples,
|
|
106
|
+
test_labels=labels,
|
|
107
|
+
test_preds=pred)
|
|
108
|
+
|
|
109
|
+
def save_results_external_dataset(out_dir, samples, label_names,
|
|
110
|
+
score, pred, test_name):
|
|
111
|
+
print('Saving model predictions to %s' % os.path.join(out_dir, test_name))
|
|
112
|
+
|
|
113
|
+
np.savez(os.path.join(out_dir, test_name),
|
|
114
|
+
test_samples=samples,
|
|
115
|
+
test_scores=score,
|
|
116
|
+
test_preds=pred,
|
|
117
|
+
label_names=label_names)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def save_additional_data(out_dir,
|
|
121
|
+
train_samples, valid_samples, test_samples,
|
|
122
|
+
test_labels, pred, score,
|
|
123
|
+
label_names, feature_names,
|
|
124
|
+
norm_mus, norm_sigmas):
|
|
125
|
+
filename = 'additional_data.npz'
|
|
126
|
+
|
|
127
|
+
print('Saving additional data to %s' % os.path.join(out_dir, filename))
|
|
128
|
+
|
|
129
|
+
np.savez(os.path.join(out_dir, filename),
|
|
130
|
+
sample_ids_train=train_samples,
|
|
131
|
+
sample_ids_valid=valid_samples,
|
|
132
|
+
sample_ids_test=test_samples,
|
|
133
|
+
test_labels=test_labels,
|
|
134
|
+
test_preds=pred,
|
|
135
|
+
test_scores=score,
|
|
136
|
+
label_names=label_names,
|
|
137
|
+
feature_names=feature_names,
|
|
138
|
+
norm_mus=norm_mus,
|
|
139
|
+
norm_sigmas=norm_sigmas)
|
|
140
|
+
"""
|
|
141
|
+
def save_additional_data_regression(out_dir,
|
|
142
|
+
train_samples, valid_samples, test_samples,
|
|
143
|
+
test_labels, pred, test_loss,feature_names):
|
|
144
|
+
|
|
145
|
+
filename = 'additional_data.npz'
|
|
146
|
+
|
|
147
|
+
print('Saving additional data to %s' % os.path.join(out_dir, filename))
|
|
148
|
+
"""
|
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
import time
|
|
2
|
+
import pprint
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
import torch
|
|
6
|
+
import torch.nn.functional as F
|
|
7
|
+
import torch.nn as nn
|
|
8
|
+
import torch.profiler as profiler
|
|
9
|
+
|
|
10
|
+
from helpers import model
|
|
11
|
+
from helpers import dataset_utils as du
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def train_step(mod_handler, device, train_dataset, train_generator,
|
|
15
|
+
mus, sigmas, normalize, results_fullpath, epoch):
|
|
16
|
+
|
|
17
|
+
task_handler = mod_handler.task_handler
|
|
18
|
+
|
|
19
|
+
# Reset to 0 batches results from previous epoch
|
|
20
|
+
task_handler.init_batches_results(train_dataset, train_generator)
|
|
21
|
+
|
|
22
|
+
# Batch start pos (to compile samples and labels)
|
|
23
|
+
bstart = 0
|
|
24
|
+
for batch, (x_batch, y_batch, samples) in enumerate(train_generator):
|
|
25
|
+
# Compile batch samples and labels
|
|
26
|
+
bend = bstart + len(samples) # batch end pos
|
|
27
|
+
task_handler.batches_results['samples'][bstart:bend] = samples
|
|
28
|
+
task_handler.batches_results['ys'][bstart:bend] = y_batch
|
|
29
|
+
|
|
30
|
+
# Send data to device
|
|
31
|
+
x_batch, y_batch = x_batch.to(device), y_batch.to(device)
|
|
32
|
+
x_batch = x_batch.float()
|
|
33
|
+
|
|
34
|
+
y_batch = task_handler.format_ybatch(y_batch)
|
|
35
|
+
|
|
36
|
+
# Replace missing values
|
|
37
|
+
du.replace_missing_values(x_batch, mus)
|
|
38
|
+
|
|
39
|
+
# Normalize
|
|
40
|
+
if normalize:
|
|
41
|
+
x_batch = du.normalize(x_batch, mus, sigmas)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
# Reset optimizer
|
|
45
|
+
optimizers = mod_handler.model.get_optimizers()
|
|
46
|
+
for optimizer in optimizers:
|
|
47
|
+
optimizer.zero_grad()
|
|
48
|
+
|
|
49
|
+
# Forward pass
|
|
50
|
+
model_out = mod_handler.model.forward(x_batch, results_fullpath,
|
|
51
|
+
epoch, batch, 'train')
|
|
52
|
+
|
|
53
|
+
# Loss
|
|
54
|
+
loss = task_handler.compute_loss(model_out, y_batch)
|
|
55
|
+
|
|
56
|
+
# Compute gradients
|
|
57
|
+
loss.backward()
|
|
58
|
+
|
|
59
|
+
# Optimize
|
|
60
|
+
for optimizer in optimizers:
|
|
61
|
+
optimizer.step()
|
|
62
|
+
|
|
63
|
+
# Loss summed over all outputs (by default Pytorch returns mean loss
|
|
64
|
+
# computed over nb of outputs)
|
|
65
|
+
loss_wo_reduction = loss.item()*len(y_batch)
|
|
66
|
+
|
|
67
|
+
# Compile batch loss wo reduction
|
|
68
|
+
task_handler.batches_results['losses_wo_reduction'][batch] = \
|
|
69
|
+
loss_wo_reduction
|
|
70
|
+
|
|
71
|
+
# Compile batch predictions
|
|
72
|
+
task_handler.update_batches_preds(model_out, y_batch, bstart, bend, batch)
|
|
73
|
+
|
|
74
|
+
# Update batch start pos
|
|
75
|
+
bstart = bend
|
|
76
|
+
|
|
77
|
+
return task_handler.batches_results.copy()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def eval_step(mod_handler, device, eval_dataset, valid_generator,
|
|
81
|
+
mus, sigmas, normalize, results_fullpath, epoch, scale=1.0):
|
|
82
|
+
|
|
83
|
+
task_handler = mod_handler.task_handler
|
|
84
|
+
|
|
85
|
+
# Reset to 0 batches results from previous epoch
|
|
86
|
+
task_handler.init_batches_results(eval_dataset, valid_generator)
|
|
87
|
+
|
|
88
|
+
# Batch start pos (to compile samples and labels)
|
|
89
|
+
bstart = 0
|
|
90
|
+
for batch, (x_batch, y_batch, samples) in enumerate(valid_generator):
|
|
91
|
+
# Compile batch samples and labels
|
|
92
|
+
bend = bstart + len(samples) # batch end pos
|
|
93
|
+
task_handler.batches_results['samples'][bstart:bend] = samples
|
|
94
|
+
task_handler.batches_results['ys'][bstart:bend] = y_batch
|
|
95
|
+
|
|
96
|
+
# Send data to device
|
|
97
|
+
x_batch, y_batch = x_batch.to(device), y_batch.to(device)
|
|
98
|
+
x_batch = x_batch.float()
|
|
99
|
+
|
|
100
|
+
y_batch = task_handler.format_ybatch(y_batch)
|
|
101
|
+
|
|
102
|
+
# Replace missing values : missing values (-1) become the SNP mean
|
|
103
|
+
du.replace_missing_values(x_batch, mus)
|
|
104
|
+
|
|
105
|
+
# Normalize (missing values become 0, because we substract the mean)
|
|
106
|
+
if normalize:
|
|
107
|
+
x_batch = du.normalize(x_batch, mus, sigmas)
|
|
108
|
+
|
|
109
|
+
# The scaling does not affect missing values because they are 0
|
|
110
|
+
x_batch = x_batch*scale
|
|
111
|
+
|
|
112
|
+
# Forward pass
|
|
113
|
+
model_out = mod_handler.model.forward(x_batch, results_fullpath,
|
|
114
|
+
epoch, batch, 'valid')
|
|
115
|
+
# Loss
|
|
116
|
+
loss = task_handler.compute_loss(model_out, y_batch)
|
|
117
|
+
|
|
118
|
+
# Loss summed over all outputs (by default Pytorch returns mean loss
|
|
119
|
+
# computed over nb of outputs)
|
|
120
|
+
loss_wo_reduction = loss.item()*len(y_batch)
|
|
121
|
+
|
|
122
|
+
# Compile batch loss wo reduction
|
|
123
|
+
task_handler.batches_results['losses_wo_reduction'][batch] = \
|
|
124
|
+
loss_wo_reduction
|
|
125
|
+
|
|
126
|
+
# Compile batch predictions
|
|
127
|
+
task_handler.update_batches_preds(model_out, y_batch, bstart, bend, batch)
|
|
128
|
+
|
|
129
|
+
# Update batch start pos
|
|
130
|
+
bstart = bend
|
|
131
|
+
|
|
132
|
+
return task_handler.batches_results.copy()
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def indep_test_step(mod_handler, device, test_dataset, test_generator,
|
|
136
|
+
mus, sigmas, normalize, results_fullpath, epoch, scale):
|
|
137
|
+
|
|
138
|
+
task_handler = mod_handler.task_handler
|
|
139
|
+
|
|
140
|
+
# Reset to 0 batches results from previous epoch
|
|
141
|
+
task_handler.init_indep_test_batches_results(test_dataset, test_generator)
|
|
142
|
+
|
|
143
|
+
# Batch start pos (to compile samples and labels)
|
|
144
|
+
bstart = 0
|
|
145
|
+
for batch, (idx, x_batch) in enumerate(test_generator):
|
|
146
|
+
# Compile batch samples and labels
|
|
147
|
+
bend = bstart + len(idx) # batch end pos
|
|
148
|
+
task_handler.batches_results['samples'][bstart:bend] = idx
|
|
149
|
+
|
|
150
|
+
# Send data to device
|
|
151
|
+
x_batch = x_batch.to(device)
|
|
152
|
+
x_batch = x_batch.float()
|
|
153
|
+
|
|
154
|
+
# Replace missing values
|
|
155
|
+
du.replace_missing_values(x_batch, mus)
|
|
156
|
+
|
|
157
|
+
# Normalize
|
|
158
|
+
if normalize:
|
|
159
|
+
x_batch = du.normalize(x_batch, mus, sigmas)
|
|
160
|
+
|
|
161
|
+
x_batch = x_batch*scale
|
|
162
|
+
|
|
163
|
+
# Forward pass
|
|
164
|
+
model_out = mod_handler.model.forward(x_batch, results_fullpath,
|
|
165
|
+
epoch, batch, 'test')
|
|
166
|
+
|
|
167
|
+
# Compile batch predictions
|
|
168
|
+
task_handler.update_indep_test_batches_preds(model_out, bstart, bend, batch)
|
|
169
|
+
|
|
170
|
+
# Update batch start pos
|
|
171
|
+
bstart = bend
|
|
172
|
+
|
|
173
|
+
return task_handler.batches_results.copy()
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def get_last_layers(comb_model, device, test_generator, set_size,
|
|
177
|
+
mus, sigmas, emb, task):
|
|
178
|
+
# Saving data seen while looping through minibatches
|
|
179
|
+
minibatch_n_right = [] #number of good classifications
|
|
180
|
+
test_pred = torch.tensor([]).to(device) #prediction of each sample
|
|
181
|
+
test_score = torch.tensor([]).to(device) #softmax values of each sample
|
|
182
|
+
test_samples = np.array([]) #test samples
|
|
183
|
+
test_ys = np.array([]) #true labels of test samples
|
|
184
|
+
|
|
185
|
+
before_last_layer = torch.tensor([])
|
|
186
|
+
out_layer = torch.tensor([])
|
|
187
|
+
|
|
188
|
+
for i, (x_batch, y_batch, samples) in enumerate(test_generator):
|
|
189
|
+
# Save samples
|
|
190
|
+
test_samples = np.concatenate([test_samples, samples])
|
|
191
|
+
# Save labels
|
|
192
|
+
test_ys = np.concatenate([test_ys, y_batch])
|
|
193
|
+
|
|
194
|
+
# Send data to device
|
|
195
|
+
x_batch, y_batch = x_batch.to(device), y_batch.to(device)
|
|
196
|
+
x_batch = x_batch.float()
|
|
197
|
+
|
|
198
|
+
# Replace missing values
|
|
199
|
+
du.replace_missing_values(x_batch, mus)
|
|
200
|
+
|
|
201
|
+
# Normalize
|
|
202
|
+
x_batch = du.normalize(x_batch, mus, sigmas)
|
|
203
|
+
|
|
204
|
+
# Forward pass
|
|
205
|
+
comb_model_before_last, comb_model_out = comb_model(emb, x_batch, save_layers=True)
|
|
206
|
+
|
|
207
|
+
# Save layers
|
|
208
|
+
before_last_layer = torch.cat((before_last_layer,comb_model_before_last.detach().cpu()),dim=0)
|
|
209
|
+
out_layer = torch.cat((out_layer,comb_model_out.detach().cpu()), dim=0)
|
|
210
|
+
|
|
211
|
+
# Predictions
|
|
212
|
+
if task == 'classification':
|
|
213
|
+
score, pred = get_predictions(comb_model_out)
|
|
214
|
+
test_pred = torch.cat((test_pred,pred), dim=-1)
|
|
215
|
+
test_score = torch.cat((test_score,score), dim=0)
|
|
216
|
+
|
|
217
|
+
# Nb of good classifications for the minibatch
|
|
218
|
+
minibatch_n_right.append(((y_batch - pred) == 0).sum().item())
|
|
219
|
+
|
|
220
|
+
elif task == 'regression':
|
|
221
|
+
test_pred = torch.cat((test_pred,comb_model_out.detach()), dim=-1)
|
|
222
|
+
|
|
223
|
+
# Total accuracy
|
|
224
|
+
test_acc = 0.0
|
|
225
|
+
if task == 'classification':
|
|
226
|
+
test_acc = np.array(minibatch_n_right).sum() / float(set_size)*100
|
|
227
|
+
|
|
228
|
+
return test_samples, test_ys, test_score, test_pred, test_acc, before_last_layer, out_layer
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def create_disc_model(comb_model, emb, device):
|
|
233
|
+
"""
|
|
234
|
+
this only works with 1 GPU or CPU
|
|
235
|
+
note: the function name is misleading. This does not create a new model. It just
|
|
236
|
+
takes the pre-existing model and
|
|
237
|
+
returns a function that performs the forward pass on it (with fixed embedding).
|
|
238
|
+
this should be okay since python passes args by reference, so even if comb_model
|
|
239
|
+
weights change, the corresponding function will also change
|
|
240
|
+
|
|
241
|
+
Finally, remember that disc_model will be in whatever mode comb_model is,
|
|
242
|
+
so if comb_model is in train mode, switch it to eval mode before calling
|
|
243
|
+
ßthe output of this.
|
|
244
|
+
"""
|
|
245
|
+
|
|
246
|
+
if torch.cuda.device_count() > 1:
|
|
247
|
+
print('warning: this only works during training/inference with 1 GPU!')
|
|
248
|
+
comb_model = comb_model.eval()
|
|
249
|
+
comb_model.to(device)
|
|
250
|
+
discrim_model = lambda x: comb_model(emb, x) # recreate discrim_model
|
|
251
|
+
return discrim_model
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def create_disc_model_multi_gpu(comb_model, emb, device, eps=1e-5, incl_softmax=False):
|
|
255
|
+
"""
|
|
256
|
+
Transforms comb_model + emb into equivalent discrim model (with fatlayer weights added as parameters)
|
|
257
|
+
This model can now be sent to multiple GPUs without any bugs
|
|
258
|
+
(cannot do multi-GPU with comb_model since dataparallel will attempt to split the embedding up,
|
|
259
|
+
which will result in size incompatibilities)
|
|
260
|
+
|
|
261
|
+
Must pass batchnorm eps seperately in case loading from Theano model!
|
|
262
|
+
Must pass incl_softmax seperately in case loading from Theano model!
|
|
263
|
+
"""
|
|
264
|
+
|
|
265
|
+
n_feats_emb = emb.size()[1] # input of aux net
|
|
266
|
+
n_feats = emb.size()[0] # input of main net
|
|
267
|
+
# Hidden layers size
|
|
268
|
+
emb_n_hidden_u = 100
|
|
269
|
+
discrim_n_hidden1_u = 100
|
|
270
|
+
discrim_n_hidden2_u = 100
|
|
271
|
+
|
|
272
|
+
# put in eval mode and send to correct device
|
|
273
|
+
comb_model = comb_model.eval().to(device)
|
|
274
|
+
emb = emb.to(device)
|
|
275
|
+
fatLayer_weights = torch.transpose(comb_model.feat_emb(emb),1,0)
|
|
276
|
+
|
|
277
|
+
# initialize discriminitive network with fatlayer as a parameter
|
|
278
|
+
# send back to cpu while loading weights
|
|
279
|
+
disc_net = model.Discrim_net(fatLayer_weights,
|
|
280
|
+
n_feats=n_feats_emb,
|
|
281
|
+
n_hidden1_u=discrim_n_hidden1_u,
|
|
282
|
+
n_hidden2_u=discrim_n_hidden2_u,
|
|
283
|
+
n_targets=26,
|
|
284
|
+
eps=eps,
|
|
285
|
+
incl_softmax=incl_softmax)
|
|
286
|
+
|
|
287
|
+
# copy over all weights
|
|
288
|
+
disc_net.out.weight.data = comb_model.disc_net.out.weight.data
|
|
289
|
+
disc_net.out.bias.data = comb_model.disc_net.out.bias.data
|
|
290
|
+
disc_net.bn2.weight.data = comb_model.disc_net.bn2.weight.data
|
|
291
|
+
disc_net.bn2.bias.data = comb_model.disc_net.bn2.bias.data
|
|
292
|
+
disc_net.bn2.running_mean = comb_model.disc_net.bn2.running_mean
|
|
293
|
+
disc_net.bn2.running_var = comb_model.disc_net.bn2.running_var
|
|
294
|
+
disc_net.hidden_2.weight.data = comb_model.disc_net.hidden_2.weight.data
|
|
295
|
+
disc_net.hidden_2.bias.data = comb_model.disc_net.hidden_2.bias.data
|
|
296
|
+
disc_net.bn1.weight.data = comb_model.disc_net.bn1.weight.data
|
|
297
|
+
disc_net.bn1.bias.data = comb_model.disc_net.bn1.bias.data
|
|
298
|
+
disc_net.bn1.running_mean = comb_model.disc_net.bn1.running_mean
|
|
299
|
+
disc_net.bn1.running_var = comb_model.disc_net.bn1.running_var
|
|
300
|
+
disc_net.hidden_1.bias.data = comb_model.disc_net.fat_bias.data
|
|
301
|
+
|
|
302
|
+
assert (disc_net.hidden_1.weight.data == fatLayer_weights).all()
|
|
303
|
+
disc_net = disc_net.eval().to('cpu')
|
|
304
|
+
|
|
305
|
+
# Now we can do dataparallel
|
|
306
|
+
if torch.cuda.device_count() > 1:
|
|
307
|
+
disc_net = nn.DataParallel(disc_net)
|
|
308
|
+
print("{} GPUs detected! Running in DataParallel mode".format(torch.cuda.device_count()))
|
|
309
|
+
disc_net.to(device)
|
|
310
|
+
return disc_net
|
|
311
|
+
|
|
312
|
+
def convert_theano_array_to_pytorch_tensor(tensor, array):
|
|
313
|
+
array_as_tensor = torch.from_numpy(array.T)
|
|
314
|
+
assert tensor.data.shape == array_as_tensor.shape
|
|
315
|
+
tensor.data = array_as_tensor
|
|
316
|
+
|
|
317
|
+
def convert_theano_array_to_pytorch_tensor_1d(tensor, array):
|
|
318
|
+
array_as_tensor = torch.from_numpy(array)
|
|
319
|
+
assert tensor.data.shape == array_as_tensor.shape
|
|
320
|
+
tensor.data = array_as_tensor
|
|
321
|
+
|
|
322
|
+
def convert_theano_bn_pytorch(bn, theano_arr_1, theano_arr_2, theano_arr_3, theano_arr_4):
|
|
323
|
+
# Idea from: https://discuss.pytorch.org/t/is-there-any-difference-between-theano-convolution-and-pytorch-convolution/10580/10
|
|
324
|
+
# Specifically how to load BN from theano to pytorch
|
|
325
|
+
#extractor += [nn.BatchNorm2d(in_channel)]
|
|
326
|
+
#extractor[-1].weight.data = torch.from_numpy(parameters[i-3])
|
|
327
|
+
#extractor[-1].bias.data = torch.from_numpy(parameters[i-2])
|
|
328
|
+
#extractor[-1].running_mean = torch.from_numpy(parameters[i-1])
|
|
329
|
+
#extractor[-1].running_var = torch.from_numpy((1./(parameters[i]**2)) - 1e-4)
|
|
330
|
+
|
|
331
|
+
convert_theano_array_to_pytorch_tensor_1d(bn.weight, theano_arr_2)
|
|
332
|
+
convert_theano_array_to_pytorch_tensor_1d(bn.bias, theano_arr_1)
|
|
333
|
+
convert_theano_array_to_pytorch_tensor_1d(bn.running_mean, theano_arr_3)
|
|
334
|
+
convert_theano_array_to_pytorch_tensor_1d(bn.running_var, (1./(theano_arr_4**2)) - 1e-4)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def load_theano_model(n_feats_emb, emb_n_hidden_u, discrim_n_hidden1_u, discrim_n_hidden2_u, n_targets, theano_weight_file, device, only_discrim_model=True):
|
|
338
|
+
# loads theano weights file into PyTorch's comb_net OR Discrim_net
|
|
339
|
+
|
|
340
|
+
theano_model_params = np.load(theano_weight_file)
|
|
341
|
+
print('theano layer shapes:')
|
|
342
|
+
for f in theano_model_params.files:
|
|
343
|
+
print(f, theano_model_params[f].shape)
|
|
344
|
+
|
|
345
|
+
comb_model = model.CombinedModel(
|
|
346
|
+
n_feats=n_feats_emb,
|
|
347
|
+
n_hidden_u=emb_n_hidden_u,
|
|
348
|
+
n_hidden1_u=discrim_n_hidden1_u,
|
|
349
|
+
n_hidden2_u=discrim_n_hidden2_u,
|
|
350
|
+
n_targets=n_targets,
|
|
351
|
+
param_init=None,
|
|
352
|
+
eps=1e-4, # Theano uses 1e-4 for batch norm instead of PyTorch default of 1e-5
|
|
353
|
+
input_dropout=0.,
|
|
354
|
+
incl_softmax=True) # theano includes softmax in output
|
|
355
|
+
|
|
356
|
+
# feat emb model
|
|
357
|
+
convert_theano_array_to_pytorch_tensor(comb_model.feat_emb.hidden_1.weight, theano_model_params['arr_0'])
|
|
358
|
+
convert_theano_array_to_pytorch_tensor(comb_model.feat_emb.hidden_2.weight, theano_model_params['arr_1'])
|
|
359
|
+
|
|
360
|
+
# embedding
|
|
361
|
+
emb = torch.tensor(theano_model_params['arr_2'])
|
|
362
|
+
|
|
363
|
+
# disc model
|
|
364
|
+
convert_theano_array_to_pytorch_tensor_1d(comb_model.disc_net.fat_bias, theano_model_params['arr_3'])
|
|
365
|
+
|
|
366
|
+
convert_theano_bn_pytorch(comb_model.disc_net.bn1,
|
|
367
|
+
theano_model_params['arr_4'],
|
|
368
|
+
theano_model_params['arr_5'],
|
|
369
|
+
theano_model_params['arr_6'],
|
|
370
|
+
theano_model_params['arr_7'])
|
|
371
|
+
|
|
372
|
+
convert_theano_array_to_pytorch_tensor(comb_model.disc_net.hidden_2.weight, theano_model_params['arr_8'])
|
|
373
|
+
convert_theano_array_to_pytorch_tensor_1d(comb_model.disc_net.hidden_2.bias, theano_model_params['arr_9'])
|
|
374
|
+
|
|
375
|
+
convert_theano_bn_pytorch(comb_model.disc_net.bn2,
|
|
376
|
+
theano_model_params['arr_10'],
|
|
377
|
+
theano_model_params['arr_11'],
|
|
378
|
+
theano_model_params['arr_12'],
|
|
379
|
+
theano_model_params['arr_13'])
|
|
380
|
+
|
|
381
|
+
convert_theano_array_to_pytorch_tensor(comb_model.disc_net.out.weight, theano_model_params['arr_14'])
|
|
382
|
+
convert_theano_array_to_pytorch_tensor_1d(comb_model.disc_net.out.bias, theano_model_params['arr_15'])
|
|
383
|
+
|
|
384
|
+
model_to_return = comb_model.to(device)
|
|
385
|
+
|
|
386
|
+
if only_discrim_model:
|
|
387
|
+
|
|
388
|
+
emb = emb.to(device)
|
|
389
|
+
# create disc_net from loaded comb_model
|
|
390
|
+
model_to_return = create_disc_model_multi_gpu(model_to_return,
|
|
391
|
+
emb, device,
|
|
392
|
+
eps=1e-4, # Theano uses 1e-4 for batch norm instead of PyTorch default of 1e-5
|
|
393
|
+
incl_softmax=True) # Theano includes softmax in model
|
|
394
|
+
|
|
395
|
+
return model_to_return
|