dietnetwork 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. dietnet/__init__.py +11 -0
  2. dietnet/_launch.py +40 -0
  3. dietnet/_upstream/Dietnet_train_and_evaluate/__init__.py +0 -0
  4. dietnet/_upstream/Dietnet_train_and_evaluate/compute_input_features_statistics.py +221 -0
  5. dietnet/_upstream/Dietnet_train_and_evaluate/generate_embedding.py +216 -0
  6. dietnet/_upstream/Dietnet_train_and_evaluate/helpers/__init__.py +0 -0
  7. dietnet/_upstream/Dietnet_train_and_evaluate/helpers/dataset_utils.py +200 -0
  8. dietnet/_upstream/Dietnet_train_and_evaluate/helpers/log_utils.py +148 -0
  9. dietnet/_upstream/Dietnet_train_and_evaluate/helpers/mainloop_utils.py +395 -0
  10. dietnet/_upstream/Dietnet_train_and_evaluate/helpers/model.py +489 -0
  11. dietnet/_upstream/Dietnet_train_and_evaluate/helpers/model_handlers.py +89 -0
  12. dietnet/_upstream/Dietnet_train_and_evaluate/helpers/task_handlers.py +361 -0
  13. dietnet/_upstream/Dietnet_train_and_evaluate/partition_data.py +331 -0
  14. dietnet/_upstream/Dietnet_train_and_evaluate/train.py +959 -0
  15. dietnet/align.py +188 -0
  16. dietnet/alphagenome.py +208 -0
  17. dietnet/cli.py +479 -0
  18. dietnet/dataset.py +181 -0
  19. dietnet/features.py +267 -0
  20. dietnet/package.py +187 -0
  21. dietnet/plink.py +163 -0
  22. dietnet/predict.py +123 -0
  23. dietnet/published.py +87 -0
  24. dietnet/py.typed +0 -0
  25. dietnet/training.py +176 -0
  26. dietnet/upstream.py +161 -0
  27. dietnet/upstream_manifest.json +53 -0
  28. dietnetwork-0.3.0.dist-info/METADATA +249 -0
  29. dietnetwork-0.3.0.dist-info/RECORD +33 -0
  30. dietnetwork-0.3.0.dist-info/WHEEL +4 -0
  31. dietnetwork-0.3.0.dist-info/entry_points.txt +2 -0
  32. dietnetwork-0.3.0.dist-info/licenses/LICENSE +28 -0
  33. dietnetwork-0.3.0.dist-info/licenses/NOTICE +15 -0
@@ -0,0 +1,148 @@
1
+ import os
2
+ import sys
3
+ from pathlib import Path
4
+ import inspect
5
+
6
+ import numpy as np
7
+
8
+ import torch
9
+
10
+
11
+ def create_dir(dir_fullpath):
12
+ """
13
+ This check if a direcotry already exists and
14
+ creates the directory if it does not
15
+ """
16
+ if not os.path.isdir(dir_fullpath):
17
+ try:
18
+ Path(dir_fullpath).mkdir(parents=True, exist_ok=True)
19
+ except OSError:
20
+ print('Creation of directory %s failed' % dir_fullpath)
21
+ sys.exit(1)
22
+
23
+
24
+ def save_exp_params(out_dir, filename, config):
25
+ """
26
+ Creates the file exp_params.log where experiment
27
+ configurations will be saved for reproducibility
28
+ """
29
+ with open(os.path.join(out_dir, filename), 'w') as f:
30
+ for k,v in config.items():
31
+ f.write(k + ':' + str(v) + '\n')
32
+
33
+
34
+ def save_model_summary(out_dir, model, criterion, optimizer):
35
+ filename = 'model_summary.log'
36
+
37
+ print('Saving model architecture summary to %s' % os.path.join(
38
+ out_dir,
39
+ filename)
40
+ )
41
+
42
+ text = '*'*50
43
+ text += '\nAuxiliary network (feature embedding network):\n'
44
+ text += '*'*50 + '\n'
45
+ text += str(model.feat_emb)
46
+ text += '\nForward function of auxiliary net:\n'
47
+ text += inspect.getsource(model.feat_emb.forward)
48
+ text += '\n' + '-'*80
49
+
50
+ text += '\n' + '*'*40
51
+ text += '\nMain network (discriminative network):\n'
52
+ text += '*'*40 + '\n'
53
+ text += str(model.disc_net)
54
+ text += '\nForward function of main net:\n'
55
+ text += inspect.getsource(model.disc_net.forward)
56
+ bias = False if model.disc_net.fat_bias is None else True
57
+ text += '\n --> Fat layer has bias param:' + str(bias) + '\n'
58
+ text += '\n' + '-'*80
59
+
60
+ text += '\n' + '*'*20
61
+ text += '\nCombined models:\n'
62
+ text += '*'*20 + '\n'
63
+ text += str(model)
64
+ text += '\nForward function of combined models:\n'
65
+ text += inspect.getsource(model.disc_net.forward)
66
+
67
+ text += '\n' + '-'*80
68
+ text += '\n' + '*'*20
69
+ text += '\nLoss and Optimizer\n'
70
+ text += '*'*20 + '\n'
71
+ text += 'Loss:\n' + str(criterion)
72
+ text += '\n\nOptimizer:\n' + str(optimizer)
73
+
74
+ with open(os.path.join(out_dir, filename), 'w') as f:
75
+ f.write(text)
76
+
77
+
78
+ def save_model_params(out_dir, model, filename='model_params.pt'):
79
+ #filename = 'model_params.pt'
80
+
81
+ print('Saving model parameters to %s' % os.path.join(out_dir, filename))
82
+
83
+ torch.save(model.state_dict(), os.path.join(out_dir, filename))
84
+
85
+
86
+ def save_results(out_dir, samples, labels, label_names, score, pred):
87
+ filename = 'model_predictions.npz'
88
+
89
+ print('Saving model predictions to %s' % os.path.join(out_dir, filename))
90
+
91
+ np.savez(os.path.join(out_dir, filename),
92
+ test_samples=samples,
93
+ test_labels=labels,
94
+ test_scores=score,
95
+ test_preds=pred,
96
+ label_names=label_names)
97
+
98
+
99
+ def save_results_regression(out_dir, samples, labels, pred):
100
+ filename = 'model_predictions.npz'
101
+
102
+ print('Saving model predictions to %s' % os.path.join(out_dir, filename))
103
+
104
+ np.savez(os.path.join(out_dir, filename),
105
+ test_samples=samples,
106
+ test_labels=labels,
107
+ test_preds=pred)
108
+
109
+ def save_results_external_dataset(out_dir, samples, label_names,
110
+ score, pred, test_name):
111
+ print('Saving model predictions to %s' % os.path.join(out_dir, test_name))
112
+
113
+ np.savez(os.path.join(out_dir, test_name),
114
+ test_samples=samples,
115
+ test_scores=score,
116
+ test_preds=pred,
117
+ label_names=label_names)
118
+
119
+
120
+ def save_additional_data(out_dir,
121
+ train_samples, valid_samples, test_samples,
122
+ test_labels, pred, score,
123
+ label_names, feature_names,
124
+ norm_mus, norm_sigmas):
125
+ filename = 'additional_data.npz'
126
+
127
+ print('Saving additional data to %s' % os.path.join(out_dir, filename))
128
+
129
+ np.savez(os.path.join(out_dir, filename),
130
+ sample_ids_train=train_samples,
131
+ sample_ids_valid=valid_samples,
132
+ sample_ids_test=test_samples,
133
+ test_labels=test_labels,
134
+ test_preds=pred,
135
+ test_scores=score,
136
+ label_names=label_names,
137
+ feature_names=feature_names,
138
+ norm_mus=norm_mus,
139
+ norm_sigmas=norm_sigmas)
140
+ """
141
+ def save_additional_data_regression(out_dir,
142
+ train_samples, valid_samples, test_samples,
143
+ test_labels, pred, test_loss,feature_names):
144
+
145
+ filename = 'additional_data.npz'
146
+
147
+ print('Saving additional data to %s' % os.path.join(out_dir, filename))
148
+ """
@@ -0,0 +1,395 @@
1
+ import time
2
+ import pprint
3
+ import numpy as np
4
+
5
+ import torch
6
+ import torch.nn.functional as F
7
+ import torch.nn as nn
8
+ import torch.profiler as profiler
9
+
10
+ from helpers import model
11
+ from helpers import dataset_utils as du
12
+
13
+
14
+ def train_step(mod_handler, device, train_dataset, train_generator,
15
+ mus, sigmas, normalize, results_fullpath, epoch):
16
+
17
+ task_handler = mod_handler.task_handler
18
+
19
+ # Reset to 0 batches results from previous epoch
20
+ task_handler.init_batches_results(train_dataset, train_generator)
21
+
22
+ # Batch start pos (to compile samples and labels)
23
+ bstart = 0
24
+ for batch, (x_batch, y_batch, samples) in enumerate(train_generator):
25
+ # Compile batch samples and labels
26
+ bend = bstart + len(samples) # batch end pos
27
+ task_handler.batches_results['samples'][bstart:bend] = samples
28
+ task_handler.batches_results['ys'][bstart:bend] = y_batch
29
+
30
+ # Send data to device
31
+ x_batch, y_batch = x_batch.to(device), y_batch.to(device)
32
+ x_batch = x_batch.float()
33
+
34
+ y_batch = task_handler.format_ybatch(y_batch)
35
+
36
+ # Replace missing values
37
+ du.replace_missing_values(x_batch, mus)
38
+
39
+ # Normalize
40
+ if normalize:
41
+ x_batch = du.normalize(x_batch, mus, sigmas)
42
+
43
+
44
+ # Reset optimizer
45
+ optimizers = mod_handler.model.get_optimizers()
46
+ for optimizer in optimizers:
47
+ optimizer.zero_grad()
48
+
49
+ # Forward pass
50
+ model_out = mod_handler.model.forward(x_batch, results_fullpath,
51
+ epoch, batch, 'train')
52
+
53
+ # Loss
54
+ loss = task_handler.compute_loss(model_out, y_batch)
55
+
56
+ # Compute gradients
57
+ loss.backward()
58
+
59
+ # Optimize
60
+ for optimizer in optimizers:
61
+ optimizer.step()
62
+
63
+ # Loss summed over all outputs (by default Pytorch returns mean loss
64
+ # computed over nb of outputs)
65
+ loss_wo_reduction = loss.item()*len(y_batch)
66
+
67
+ # Compile batch loss wo reduction
68
+ task_handler.batches_results['losses_wo_reduction'][batch] = \
69
+ loss_wo_reduction
70
+
71
+ # Compile batch predictions
72
+ task_handler.update_batches_preds(model_out, y_batch, bstart, bend, batch)
73
+
74
+ # Update batch start pos
75
+ bstart = bend
76
+
77
+ return task_handler.batches_results.copy()
78
+
79
+
80
+ def eval_step(mod_handler, device, eval_dataset, valid_generator,
81
+ mus, sigmas, normalize, results_fullpath, epoch, scale=1.0):
82
+
83
+ task_handler = mod_handler.task_handler
84
+
85
+ # Reset to 0 batches results from previous epoch
86
+ task_handler.init_batches_results(eval_dataset, valid_generator)
87
+
88
+ # Batch start pos (to compile samples and labels)
89
+ bstart = 0
90
+ for batch, (x_batch, y_batch, samples) in enumerate(valid_generator):
91
+ # Compile batch samples and labels
92
+ bend = bstart + len(samples) # batch end pos
93
+ task_handler.batches_results['samples'][bstart:bend] = samples
94
+ task_handler.batches_results['ys'][bstart:bend] = y_batch
95
+
96
+ # Send data to device
97
+ x_batch, y_batch = x_batch.to(device), y_batch.to(device)
98
+ x_batch = x_batch.float()
99
+
100
+ y_batch = task_handler.format_ybatch(y_batch)
101
+
102
+ # Replace missing values : missing values (-1) become the SNP mean
103
+ du.replace_missing_values(x_batch, mus)
104
+
105
+ # Normalize (missing values become 0, because we substract the mean)
106
+ if normalize:
107
+ x_batch = du.normalize(x_batch, mus, sigmas)
108
+
109
+ # The scaling does not affect missing values because they are 0
110
+ x_batch = x_batch*scale
111
+
112
+ # Forward pass
113
+ model_out = mod_handler.model.forward(x_batch, results_fullpath,
114
+ epoch, batch, 'valid')
115
+ # Loss
116
+ loss = task_handler.compute_loss(model_out, y_batch)
117
+
118
+ # Loss summed over all outputs (by default Pytorch returns mean loss
119
+ # computed over nb of outputs)
120
+ loss_wo_reduction = loss.item()*len(y_batch)
121
+
122
+ # Compile batch loss wo reduction
123
+ task_handler.batches_results['losses_wo_reduction'][batch] = \
124
+ loss_wo_reduction
125
+
126
+ # Compile batch predictions
127
+ task_handler.update_batches_preds(model_out, y_batch, bstart, bend, batch)
128
+
129
+ # Update batch start pos
130
+ bstart = bend
131
+
132
+ return task_handler.batches_results.copy()
133
+
134
+
135
+ def indep_test_step(mod_handler, device, test_dataset, test_generator,
136
+ mus, sigmas, normalize, results_fullpath, epoch, scale):
137
+
138
+ task_handler = mod_handler.task_handler
139
+
140
+ # Reset to 0 batches results from previous epoch
141
+ task_handler.init_indep_test_batches_results(test_dataset, test_generator)
142
+
143
+ # Batch start pos (to compile samples and labels)
144
+ bstart = 0
145
+ for batch, (idx, x_batch) in enumerate(test_generator):
146
+ # Compile batch samples and labels
147
+ bend = bstart + len(idx) # batch end pos
148
+ task_handler.batches_results['samples'][bstart:bend] = idx
149
+
150
+ # Send data to device
151
+ x_batch = x_batch.to(device)
152
+ x_batch = x_batch.float()
153
+
154
+ # Replace missing values
155
+ du.replace_missing_values(x_batch, mus)
156
+
157
+ # Normalize
158
+ if normalize:
159
+ x_batch = du.normalize(x_batch, mus, sigmas)
160
+
161
+ x_batch = x_batch*scale
162
+
163
+ # Forward pass
164
+ model_out = mod_handler.model.forward(x_batch, results_fullpath,
165
+ epoch, batch, 'test')
166
+
167
+ # Compile batch predictions
168
+ task_handler.update_indep_test_batches_preds(model_out, bstart, bend, batch)
169
+
170
+ # Update batch start pos
171
+ bstart = bend
172
+
173
+ return task_handler.batches_results.copy()
174
+
175
+
176
+ def get_last_layers(comb_model, device, test_generator, set_size,
177
+ mus, sigmas, emb, task):
178
+ # Saving data seen while looping through minibatches
179
+ minibatch_n_right = [] #number of good classifications
180
+ test_pred = torch.tensor([]).to(device) #prediction of each sample
181
+ test_score = torch.tensor([]).to(device) #softmax values of each sample
182
+ test_samples = np.array([]) #test samples
183
+ test_ys = np.array([]) #true labels of test samples
184
+
185
+ before_last_layer = torch.tensor([])
186
+ out_layer = torch.tensor([])
187
+
188
+ for i, (x_batch, y_batch, samples) in enumerate(test_generator):
189
+ # Save samples
190
+ test_samples = np.concatenate([test_samples, samples])
191
+ # Save labels
192
+ test_ys = np.concatenate([test_ys, y_batch])
193
+
194
+ # Send data to device
195
+ x_batch, y_batch = x_batch.to(device), y_batch.to(device)
196
+ x_batch = x_batch.float()
197
+
198
+ # Replace missing values
199
+ du.replace_missing_values(x_batch, mus)
200
+
201
+ # Normalize
202
+ x_batch = du.normalize(x_batch, mus, sigmas)
203
+
204
+ # Forward pass
205
+ comb_model_before_last, comb_model_out = comb_model(emb, x_batch, save_layers=True)
206
+
207
+ # Save layers
208
+ before_last_layer = torch.cat((before_last_layer,comb_model_before_last.detach().cpu()),dim=0)
209
+ out_layer = torch.cat((out_layer,comb_model_out.detach().cpu()), dim=0)
210
+
211
+ # Predictions
212
+ if task == 'classification':
213
+ score, pred = get_predictions(comb_model_out)
214
+ test_pred = torch.cat((test_pred,pred), dim=-1)
215
+ test_score = torch.cat((test_score,score), dim=0)
216
+
217
+ # Nb of good classifications for the minibatch
218
+ minibatch_n_right.append(((y_batch - pred) == 0).sum().item())
219
+
220
+ elif task == 'regression':
221
+ test_pred = torch.cat((test_pred,comb_model_out.detach()), dim=-1)
222
+
223
+ # Total accuracy
224
+ test_acc = 0.0
225
+ if task == 'classification':
226
+ test_acc = np.array(minibatch_n_right).sum() / float(set_size)*100
227
+
228
+ return test_samples, test_ys, test_score, test_pred, test_acc, before_last_layer, out_layer
229
+
230
+
231
+
232
+ def create_disc_model(comb_model, emb, device):
233
+ """
234
+ this only works with 1 GPU or CPU
235
+ note: the function name is misleading. This does not create a new model. It just
236
+ takes the pre-existing model and
237
+ returns a function that performs the forward pass on it (with fixed embedding).
238
+ this should be okay since python passes args by reference, so even if comb_model
239
+ weights change, the corresponding function will also change
240
+
241
+ Finally, remember that disc_model will be in whatever mode comb_model is,
242
+ so if comb_model is in train mode, switch it to eval mode before calling
243
+ ßthe output of this.
244
+ """
245
+
246
+ if torch.cuda.device_count() > 1:
247
+ print('warning: this only works during training/inference with 1 GPU!')
248
+ comb_model = comb_model.eval()
249
+ comb_model.to(device)
250
+ discrim_model = lambda x: comb_model(emb, x) # recreate discrim_model
251
+ return discrim_model
252
+
253
+
254
+ def create_disc_model_multi_gpu(comb_model, emb, device, eps=1e-5, incl_softmax=False):
255
+ """
256
+ Transforms comb_model + emb into equivalent discrim model (with fatlayer weights added as parameters)
257
+ This model can now be sent to multiple GPUs without any bugs
258
+ (cannot do multi-GPU with comb_model since dataparallel will attempt to split the embedding up,
259
+ which will result in size incompatibilities)
260
+
261
+ Must pass batchnorm eps seperately in case loading from Theano model!
262
+ Must pass incl_softmax seperately in case loading from Theano model!
263
+ """
264
+
265
+ n_feats_emb = emb.size()[1] # input of aux net
266
+ n_feats = emb.size()[0] # input of main net
267
+ # Hidden layers size
268
+ emb_n_hidden_u = 100
269
+ discrim_n_hidden1_u = 100
270
+ discrim_n_hidden2_u = 100
271
+
272
+ # put in eval mode and send to correct device
273
+ comb_model = comb_model.eval().to(device)
274
+ emb = emb.to(device)
275
+ fatLayer_weights = torch.transpose(comb_model.feat_emb(emb),1,0)
276
+
277
+ # initialize discriminitive network with fatlayer as a parameter
278
+ # send back to cpu while loading weights
279
+ disc_net = model.Discrim_net(fatLayer_weights,
280
+ n_feats=n_feats_emb,
281
+ n_hidden1_u=discrim_n_hidden1_u,
282
+ n_hidden2_u=discrim_n_hidden2_u,
283
+ n_targets=26,
284
+ eps=eps,
285
+ incl_softmax=incl_softmax)
286
+
287
+ # copy over all weights
288
+ disc_net.out.weight.data = comb_model.disc_net.out.weight.data
289
+ disc_net.out.bias.data = comb_model.disc_net.out.bias.data
290
+ disc_net.bn2.weight.data = comb_model.disc_net.bn2.weight.data
291
+ disc_net.bn2.bias.data = comb_model.disc_net.bn2.bias.data
292
+ disc_net.bn2.running_mean = comb_model.disc_net.bn2.running_mean
293
+ disc_net.bn2.running_var = comb_model.disc_net.bn2.running_var
294
+ disc_net.hidden_2.weight.data = comb_model.disc_net.hidden_2.weight.data
295
+ disc_net.hidden_2.bias.data = comb_model.disc_net.hidden_2.bias.data
296
+ disc_net.bn1.weight.data = comb_model.disc_net.bn1.weight.data
297
+ disc_net.bn1.bias.data = comb_model.disc_net.bn1.bias.data
298
+ disc_net.bn1.running_mean = comb_model.disc_net.bn1.running_mean
299
+ disc_net.bn1.running_var = comb_model.disc_net.bn1.running_var
300
+ disc_net.hidden_1.bias.data = comb_model.disc_net.fat_bias.data
301
+
302
+ assert (disc_net.hidden_1.weight.data == fatLayer_weights).all()
303
+ disc_net = disc_net.eval().to('cpu')
304
+
305
+ # Now we can do dataparallel
306
+ if torch.cuda.device_count() > 1:
307
+ disc_net = nn.DataParallel(disc_net)
308
+ print("{} GPUs detected! Running in DataParallel mode".format(torch.cuda.device_count()))
309
+ disc_net.to(device)
310
+ return disc_net
311
+
312
+ def convert_theano_array_to_pytorch_tensor(tensor, array):
313
+ array_as_tensor = torch.from_numpy(array.T)
314
+ assert tensor.data.shape == array_as_tensor.shape
315
+ tensor.data = array_as_tensor
316
+
317
+ def convert_theano_array_to_pytorch_tensor_1d(tensor, array):
318
+ array_as_tensor = torch.from_numpy(array)
319
+ assert tensor.data.shape == array_as_tensor.shape
320
+ tensor.data = array_as_tensor
321
+
322
+ def convert_theano_bn_pytorch(bn, theano_arr_1, theano_arr_2, theano_arr_3, theano_arr_4):
323
+ # Idea from: https://discuss.pytorch.org/t/is-there-any-difference-between-theano-convolution-and-pytorch-convolution/10580/10
324
+ # Specifically how to load BN from theano to pytorch
325
+ #extractor += [nn.BatchNorm2d(in_channel)]
326
+ #extractor[-1].weight.data = torch.from_numpy(parameters[i-3])
327
+ #extractor[-1].bias.data = torch.from_numpy(parameters[i-2])
328
+ #extractor[-1].running_mean = torch.from_numpy(parameters[i-1])
329
+ #extractor[-1].running_var = torch.from_numpy((1./(parameters[i]**2)) - 1e-4)
330
+
331
+ convert_theano_array_to_pytorch_tensor_1d(bn.weight, theano_arr_2)
332
+ convert_theano_array_to_pytorch_tensor_1d(bn.bias, theano_arr_1)
333
+ convert_theano_array_to_pytorch_tensor_1d(bn.running_mean, theano_arr_3)
334
+ convert_theano_array_to_pytorch_tensor_1d(bn.running_var, (1./(theano_arr_4**2)) - 1e-4)
335
+
336
+
337
+ def load_theano_model(n_feats_emb, emb_n_hidden_u, discrim_n_hidden1_u, discrim_n_hidden2_u, n_targets, theano_weight_file, device, only_discrim_model=True):
338
+ # loads theano weights file into PyTorch's comb_net OR Discrim_net
339
+
340
+ theano_model_params = np.load(theano_weight_file)
341
+ print('theano layer shapes:')
342
+ for f in theano_model_params.files:
343
+ print(f, theano_model_params[f].shape)
344
+
345
+ comb_model = model.CombinedModel(
346
+ n_feats=n_feats_emb,
347
+ n_hidden_u=emb_n_hidden_u,
348
+ n_hidden1_u=discrim_n_hidden1_u,
349
+ n_hidden2_u=discrim_n_hidden2_u,
350
+ n_targets=n_targets,
351
+ param_init=None,
352
+ eps=1e-4, # Theano uses 1e-4 for batch norm instead of PyTorch default of 1e-5
353
+ input_dropout=0.,
354
+ incl_softmax=True) # theano includes softmax in output
355
+
356
+ # feat emb model
357
+ convert_theano_array_to_pytorch_tensor(comb_model.feat_emb.hidden_1.weight, theano_model_params['arr_0'])
358
+ convert_theano_array_to_pytorch_tensor(comb_model.feat_emb.hidden_2.weight, theano_model_params['arr_1'])
359
+
360
+ # embedding
361
+ emb = torch.tensor(theano_model_params['arr_2'])
362
+
363
+ # disc model
364
+ convert_theano_array_to_pytorch_tensor_1d(comb_model.disc_net.fat_bias, theano_model_params['arr_3'])
365
+
366
+ convert_theano_bn_pytorch(comb_model.disc_net.bn1,
367
+ theano_model_params['arr_4'],
368
+ theano_model_params['arr_5'],
369
+ theano_model_params['arr_6'],
370
+ theano_model_params['arr_7'])
371
+
372
+ convert_theano_array_to_pytorch_tensor(comb_model.disc_net.hidden_2.weight, theano_model_params['arr_8'])
373
+ convert_theano_array_to_pytorch_tensor_1d(comb_model.disc_net.hidden_2.bias, theano_model_params['arr_9'])
374
+
375
+ convert_theano_bn_pytorch(comb_model.disc_net.bn2,
376
+ theano_model_params['arr_10'],
377
+ theano_model_params['arr_11'],
378
+ theano_model_params['arr_12'],
379
+ theano_model_params['arr_13'])
380
+
381
+ convert_theano_array_to_pytorch_tensor(comb_model.disc_net.out.weight, theano_model_params['arr_14'])
382
+ convert_theano_array_to_pytorch_tensor_1d(comb_model.disc_net.out.bias, theano_model_params['arr_15'])
383
+
384
+ model_to_return = comb_model.to(device)
385
+
386
+ if only_discrim_model:
387
+
388
+ emb = emb.to(device)
389
+ # create disc_net from loaded comb_model
390
+ model_to_return = create_disc_model_multi_gpu(model_to_return,
391
+ emb, device,
392
+ eps=1e-4, # Theano uses 1e-4 for batch norm instead of PyTorch default of 1e-5
393
+ incl_softmax=True) # Theano includes softmax in model
394
+
395
+ return model_to_return