Myosotis-Researches 0.1.8__py3-none-any.whl → 0.1.10__py3-none-any.whl

myosotis_researches/CcGAN/train_128_output_10/pretrain_CNN_class.py

@@ -1,251 +0,0 @@
-"""
-
-Pre-train a CNN on the whole dataset for evaluation purpose
-
-"""
-import os
-import argparse
-import shutil
-import timeit
-
-import torch
-import torchvision
-import torchvision.transforms as transforms
-import numpy as np
-import torch.nn as nn
-import torch.backends.cudnn as cudnn
-import random
-import matplotlib.pyplot as plt
-import matplotlib as mpl
-from torch import autograd
-from torchvision.utils import save_image
-import csv
-from tqdm import tqdm
-import gc
-import h5py
-
-from models import *
-from utils import IMGs_dataset
-
-
-#############################
-# Settings
-#############################
-
-parser = argparse.ArgumentParser(description='Pre-train CNNs')
-parser.add_argument('--root_path', type=str, default='')
-parser.add_argument('--data_path', type=str, default='')
-parser.add_argument('--num_workers', type=int, default=0)
-parser.add_argument('--CNN', type=str, default='ResNet34_class',
-                    help='CNN for training; ResNetXX')
-parser.add_argument('--epochs', type=int, default=200, metavar='N',
-                    help='number of epochs to train CNNs (default: 200)')
-parser.add_argument('--batch_size_train', type=int, default=128, metavar='N',
-                    help='input batch size for training')
-parser.add_argument('--batch_size_valid', type=int, default=10, metavar='N',
-                    help='input batch size for testing')
-parser.add_argument('--base_lr', type=float, default=0.01,
-                    help='learning rate, default=0.1')
-parser.add_argument('--weight_dacay', type=float, default=1e-4,
-                    help='Weigth decay, default=1e-4')
-parser.add_argument('--seed', type=int, default=2020, metavar='S',
-                    help='random seed (default: 1)')
-parser.add_argument('--CVMode', action='store_true', default=False,
-                    help='CV mode?')
-parser.add_argument('--valid_proport', type=float, default=0.1,
-                    help='Proportion of validation samples')
-parser.add_argument('--img_size', type=int, default=128, metavar='N')
-parser.add_argument('--min_label', type=float, default=0.0)
-parser.add_argument('--max_label', type=float, default=90.0)
-args = parser.parse_args()
-
-
-wd = args.root_path
-os.chdir(wd)
-from ..models_128 import *
-from .utils import IMGs_dataset
-
-# cuda
-device = torch.device("cuda")
-ngpu = torch.cuda.device_count()  # number of gpus
-
-# random seed
-random.seed(args.seed)
-torch.manual_seed(args.seed)
-torch.backends.cudnn.deterministic = True
-np.random.seed(args.seed)
-
-# directories for checkpoint, images and log files
-save_models_folder = wd + '/output/eval_models/'
-os.makedirs(save_models_folder, exist_ok=True)
-
-
-# data loader
-data_filename = args.data_path + '/Ra_' + str(args.img_size) + 'x' + str(args.img_size) + '.h5'
-hf = h5py.File(data_filename, 'r')
-angles = hf['labels'][:]
-angles = angles.astype(float)
-labels = hf['types'][:]
-images = hf['images'][:]
-hf.close()
-num_classes = len(set(labels))
-assert max(labels)==num_classes-1
-
-q1 = args.min_label
-q2 = args.max_label
-indx = np.where((angles>q1)*(angles<q2)==True)[0]
-angles = angles[indx]
-labels = labels[indx]
-images = images[indx]
-assert len(labels)==len(images)
-assert len(angles)==len(images)
-
-
-# define training (and validaiton) set
-if args.CVMode:
-    for i in range(num_classes):
-        indx_i = np.where(labels==i)[0] #i-th class
-        np.random.shuffle(indx_i)
-        num_imgs_all_i = len(indx_i)
-        num_imgs_valid_i = int(num_imgs_all_i*args.valid_proport)
-        if i == 0:
-            indx_valid = indx_i[0:num_imgs_valid_i]
-            indx_train = indx_i[num_imgs_valid_i:]
-        else:
-            indx_valid = np.concatenate((indx_valid, indx_i[0:num_imgs_valid_i]))
-            indx_train = np.concatenate((indx_train, indx_i[num_imgs_valid_i:]))
-    #end for i
-    trainset = IMGs_dataset(images[indx_train], labels[indx_train], normalize=True)
-    trainloader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size_train, shuffle=True, num_workers=args.num_workers)
-    validset = IMGs_dataset(images[indx_valid], labels[indx_valid], normalize=True)
-    validloader = torch.utils.data.DataLoader(validset, batch_size=args.batch_size_valid, shuffle=False, num_workers=args.num_workers)
-else:
-    trainset = IMGs_dataset(images, labels, normalize=True)
-    trainloader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size_train, shuffle=True, num_workers=args.num_workers)
-
-
-
-###########################################################################################################
-# Necessary functions
-###########################################################################################################
-
-#initialize CNNs
-def net_initialization(Pretrained_CNN_Name, ngpu = 1, num_classes=num_classes):
-    if Pretrained_CNN_Name == "ResNet18_class":
-        net = ResNet18_class_eval(num_classes=num_classes, ngpu = ngpu)
-    elif Pretrained_CNN_Name == "ResNet34_class":
-        net = ResNet34_class_eval(num_classes=num_classes, ngpu = ngpu)
-    elif Pretrained_CNN_Name == "ResNet50_class":
-        net = ResNet50_class_eval(num_classes=num_classes, ngpu = ngpu)
-    elif Pretrained_CNN_Name == "ResNet101_class":
-        net = ResNet101_class_eval(num_classes=num_classes, ngpu = ngpu)
-
-    net_name = 'PreCNNForEvalGANs_' + Pretrained_CNN_Name #get the net's name
-    net = net.to(device)
-
-    return net, net_name
-
-#adjust CNN learning rate
-def adjust_learning_rate(optimizer, epoch, BASE_LR_CNN):
-    lr = BASE_LR_CNN
-    # if epoch >= 35:
-    #     lr /= 10
-    # if epoch >= 70:
-    #     lr /= 10
-    if epoch >= 50:
-        lr /= 10
-    if epoch >= 120:
-        lr /= 10
-    for param_group in optimizer.param_groups:
-        param_group['lr'] = lr
-
-
-def train_CNN():
-
-    start_tmp = timeit.default_timer()
-    for epoch in range(args.epochs):
-        net.train()
-        train_loss = 0
-        adjust_learning_rate(optimizer, epoch, args.base_lr)
-        for batch_idx, (batch_train_images, batch_train_labels) in enumerate(trainloader):
-
-            # batch_train_images = nn.functional.interpolate(batch_train_images, size = (299,299), scale_factor=None, mode='bilinear', align_corners=False)
-
-            batch_train_images = batch_train_images.type(torch.float).cuda()
-            batch_train_labels = batch_train_labels.type(torch.long).cuda()
-
-            #Forward pass
-            outputs,_ = net(batch_train_images)
-            loss = criterion(outputs, batch_train_labels)
-
-            #backward pass
-            optimizer.zero_grad()
-            loss.backward()
-            optimizer.step()
-
-            train_loss += loss.cpu().item()
-        #end for batch_idx
-        train_loss = train_loss / len(trainloader)
-
-        if args.CVMode:
-            valid_acc = valid_CNN(verbose=False)
-            print('CNN: [epoch %d/%d] train_loss:%f valid_acc:%f Time: %.4f' % (epoch+1, args.epochs, train_loss, valid_acc, timeit.default_timer()-start_tmp))
-        else:
-            print('CNN: [epoch %d/%d] train_loss:%f Time: %.4f' % (epoch+1, args.epochs, train_loss, timeit.default_timer()-start_tmp))
-    #end for epoch
-
-    return net, optimizer
-
-
-if args.CVMode:
-    def valid_CNN(verbose=True):
-        net.eval()  # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance)
-        with torch.no_grad():
-            correct = 0
-            total = 0
-            for batch_idx, (images, labels) in enumerate(validloader):
-                images = images.type(torch.float).cuda()
-                labels = labels.type(torch.long).cuda()
-                outputs,_ = net(images)
-                _, predicted = torch.max(outputs.data, 1)
-                total += labels.size(0)
-                correct += (predicted == labels).sum().item()
-            if verbose:
-                print('Valid Accuracy of the model on the validation set: {} %'.format(100.0 * correct / total))
-        return 100.0 * correct / total
-
-
-
-###########################################################################################################
-# Training and Testing
-###########################################################################################################
-# model initialization
-net, net_name = net_initialization(args.CNN, ngpu = ngpu, num_classes = num_classes)
-criterion = nn.CrossEntropyLoss()
-optimizer = torch.optim.SGD(net.parameters(), lr = args.base_lr, momentum= 0.9, weight_decay=args.weight_dacay)
-
-filename_ckpt = save_models_folder + '/ckpt_{}_epoch_{}_seed_{}_classify_{}_chair_types_CVMode_{}.pth'.format(net_name, args.epochs, args.seed, num_classes, args.CVMode)
-
-# training
-if not os.path.isfile(filename_ckpt):
-    # TRAIN CNN
-    print("\n Begin training CNN: ")
-    start = timeit.default_timer()
-    net, optimizer = train_CNN()
-    stop = timeit.default_timer()
-    print("Time elapses: {}s".format(stop - start))
-    # save model
-    torch.save({
-    'net_state_dict': net.state_dict(),
-    }, filename_ckpt)
-else:
-    print("\n Ckpt already exists")
-    print("\n Loading...")
-    checkpoint = torch.load(filename_ckpt)
-    net.load_state_dict(checkpoint['net_state_dict'])
-torch.cuda.empty_cache()#release GPU mem which is  not references
-
-if args.CVMode:
-    #validation
-    _ = valid_CNN(True)
-    torch.cuda.empty_cache()

myosotis_researches/CcGAN/train_128_output_10/pretrain_CNN_regre.py

@@ -1,255 +0,0 @@
-"""
-
-Pre-train a CNN on the whole dataset for evaluation purpose
-
-"""
-import os
-import argparse
-import shutil
-import timeit
-import torch
-import torchvision
-import torchvision.transforms as transforms
-import numpy as np
-import torch.nn as nn
-import torch.backends.cudnn as cudnn
-import random
-import matplotlib.pyplot as plt
-import matplotlib as mpl
-from torch import autograd
-from torchvision.utils import save_image
-import csv
-from tqdm import tqdm
-import gc
-import h5py
-
-
-#############################
-# Settings
-#############################
-
-parser = argparse.ArgumentParser(description='Pre-train CNNs')
-parser.add_argument('--root_path', type=str, default='')
-parser.add_argument('--data_path', type=str, default='')
-parser.add_argument('--num_workers', type=int, default=0)
-parser.add_argument('--CNN', type=str, default='ResNet34_regre',
-                    help='CNN for training; ResNetXX')
-parser.add_argument('--epochs', type=int, default=200, metavar='N',
-                    help='number of epochs to train CNNs (default: 200)')
-parser.add_argument('--batch_size_train', type=int, default=256, metavar='N',
-                    help='input batch size for training')
-parser.add_argument('--batch_size_valid', type=int, default=10, metavar='N',
-                    help='input batch size for testing')
-parser.add_argument('--base_lr', type=float, default=0.01,
-                    help='learning rate, default=0.1')
-parser.add_argument('--weight_dacay', type=float, default=1e-4,
-                    help='Weigth decay, default=1e-4')
-parser.add_argument('--seed', type=int, default=2020, metavar='S',
-                    help='random seed (default: 1)')
-parser.add_argument('--CVMode', action='store_true', default=False,
-                   help='CV mode?')
-parser.add_argument('--img_size', type=int, default=128, metavar='N')
-parser.add_argument('--min_label', type=float, default=0.0)
-parser.add_argument('--max_label', type=float, default=90.0)
-args = parser.parse_args()
-
-
-wd = args.root_path
-os.chdir(wd)
-from ..models_128 import *
-from .utils import IMGs_dataset
-
-# cuda
-device = torch.device("cuda")
-ngpu = torch.cuda.device_count()  # number of gpus
-
-# random seed
-random.seed(args.seed)
-torch.manual_seed(args.seed)
-torch.backends.cudnn.deterministic = True
-np.random.seed(args.seed)
-
-# directories for checkpoint, images and log files
-save_models_folder = wd + '/output/eval_models/'
-os.makedirs(save_models_folder, exist_ok=True)
-
-
-###########################################################################################################
-# Data
-###########################################################################################################
-# data loader
-data_filename = args.data_path + '/Ra_' + str(args.img_size) + 'x' + str(args.img_size) + '.h5'
-hf = h5py.File(data_filename, 'r')
-labels = hf['labels'][:]
-labels = labels.astype(float)
-images = hf['images'][:]
-hf.close()
-N_all = len(images)
-assert len(images) == len(labels)
-
-q1 = args.min_label
-q2 = args.max_label
-indx = np.where((labels>q1)*(labels<q2)==True)[0]
-labels = labels[indx]
-images = images[indx]
-assert len(labels)==len(images)
-
-
-# normalize to [0, 1]
-#min_label = np.min(labels)
-#labels += np.abs(min_label)
-#max_label = np.max(labels)
-#labels /= max_label
-labels /= args.max_label
-
-
-# define training and validation sets
-if args.CVMode:
-    #80% Training; 20% valdation
-    indx_all = np.arange(len(labels))
-    np.random.shuffle(indx_all)
-    indx_valid = indx_all[0:int(0.2*len(labels))]
-    indx_train = indx_all[int(0.2*len(labels)):]
-
-    trainset = IMGs_dataset(images[indx_train], labels[indx_train], normalize=True)
-    trainloader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size_train, shuffle=True, num_workers=args.num_workers)
-    validset = IMGs_dataset(images[indx_valid], labels[indx_valid], normalize=True)
-    validloader = torch.utils.data.DataLoader(validset, batch_size=args.batch_size_valid, shuffle=False, num_workers=args.num_workers)
-
-else:
-    trainset = IMGs_dataset(images, labels, normalize=True)
-    trainloader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size_train, shuffle=True, num_workers=args.num_workers)
-
-
-
-
-###########################################################################################################
-# Necessary functions
-###########################################################################################################
-
-#initialize CNNs
-def net_initialization(Pretrained_CNN_Name, ngpu = 1):
-    if Pretrained_CNN_Name == "ResNet18_regre":
-        net = ResNet18_regre_eval(ngpu = ngpu)
-    elif Pretrained_CNN_Name == "ResNet34_regre":
-        net = ResNet34_regre_eval(ngpu = ngpu)
-    elif Pretrained_CNN_Name == "ResNet50_regre":
-        net = ResNet50_regre_eval(ngpu = ngpu)
-    elif Pretrained_CNN_Name == "ResNet101_regre":
-        net = ResNet101_regre_eval(ngpu = ngpu)
-
-    net_name = 'PreCNNForEvalGANs_' + Pretrained_CNN_Name #get the net's name
-    net = net.to(device)
-
-    return net, net_name
-
-#adjust CNN learning rate
-def adjust_learning_rate(optimizer, epoch, BASE_LR_CNN):
-    lr = BASE_LR_CNN
-    # if epoch >= 35:
-    #     lr /= 10
-    # if epoch >= 70:
-    #     lr /= 10
-    if epoch >= 50:
-        lr /= 10
-    if epoch >= 120:
-        lr /= 10
-    for param_group in optimizer.param_groups:
-        param_group['lr'] = lr
-
-
-def train_CNN():
-
-    start_tmp = timeit.default_timer()
-    for epoch in range(args.epochs):
-        net.train()
-        train_loss = 0
-        adjust_learning_rate(optimizer, epoch, args.base_lr)
-        for batch_idx, (batch_train_images, batch_train_labels) in enumerate(trainloader):
-
-            # batch_train_images = nn.functional.interpolate(batch_train_images, size = (299,299), scale_factor=None, mode='bilinear', align_corners=False)
-
-            batch_train_images = batch_train_images.type(torch.float).cuda()
-            batch_train_labels = batch_train_labels.type(torch.float).view(-1,1).cuda()
-
-            #Forward pass
-            outputs,_ = net(batch_train_images)
-            loss = criterion(outputs, batch_train_labels)
-
-            #backward pass
-            optimizer.zero_grad()
-            loss.backward()
-            optimizer.step()
-
-            train_loss += loss.cpu().item()
-        #end for batch_idx
-        train_loss = train_loss / len(trainloader)
-
-        if args.CVMode:
-            valid_loss = valid_CNN(verbose=False)
-            print('CNN: [epoch %d/%d] train_loss:%f valid_loss (avg_abs):%f Time: %.4f' % (epoch+1, args.epochs, train_loss, valid_loss, timeit.default_timer()-start_tmp))
-        else:
-            print('CNN: [epoch %d/%d] train_loss:%f Time:%.4f' % (epoch+1, args.epochs, train_loss, timeit.default_timer()-start_tmp))
-    #end for epoch
-
-    return net, optimizer
-
-if args.CVMode:
-    def valid_CNN(verbose=True):
-        net.eval()  # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance)
-        with torch.no_grad():
-            abs_diff_avg = 0
-            total = 0
-            for batch_idx, (images, labels) in enumerate(validloader):
-                images = images.type(torch.float).cuda()
-                labels = labels.type(torch.float).view(-1).cpu().numpy()
-                outputs,_ = net(images)
-                outputs = outputs.view(-1).cpu().numpy()
-                labels = labels * args.max_label
-                outputs = outputs * args.max_label
-                abs_diff_avg += np.sum(np.abs(labels-outputs))
-                total += len(labels)
-
-            if verbose:
-                print('Validation Average Absolute Difference: {}'.format(abs_diff_avg/total))
-        return abs_diff_avg/total
-
-
-
-###########################################################################################################
-# Training and validation
-###########################################################################################################
-
-
-# model initialization
-net, net_name = net_initialization(args.CNN, ngpu = ngpu)
-criterion = nn.MSELoss()
-optimizer = torch.optim.SGD(net.parameters(), lr = args.base_lr, momentum= 0.9, weight_decay=args.weight_dacay)
-
-filename_ckpt = save_models_folder + '/ckpt_{}_epoch_{}_seed_{}_CVMode_{}.pth'.format(net_name, args.epochs, args.seed, args.CVMode)
-
-
-# training
-if not os.path.isfile(filename_ckpt):
-    # TRAIN CNN
-    print("\n Begin training CNN: ")
-    start = timeit.default_timer()
-    net, optimizer = train_CNN()
-    stop = timeit.default_timer()
-    print("Time elapses: {}s".format(stop - start))
-    # save model
-    torch.save({
-    'net_state_dict': net.state_dict(),
-    }, filename_ckpt)
-else:
-    print("\n Ckpt already exists")
-    print("\n Loading...")
-    checkpoint = torch.load(filename_ckpt)
-    net.load_state_dict(checkpoint['net_state_dict'])
-torch.cuda.empty_cache()#release GPU mem which is  not references
-
-
-if args.CVMode:
-    #validation
-    _ = valid_CNN(True)
-    torch.cuda.empty_cache()

myosotis_researches/CcGAN/train_128_output_10/train_net_for_label_embed.py

@@ -1,181 +0,0 @@
-
-import torch
-import torch.nn as nn
-from torchvision.utils import save_image
-import numpy as np
-import os
-import timeit
-from PIL import Image
-
-
-
-
-#-------------------------------------------------------------
-def train_net_embed(net, net_name, trainloader, testloader, epochs=200, resume_epoch = 0, lr_base=0.01, lr_decay_factor=0.1, lr_decay_epochs=[80, 140], weight_decay=1e-4, path_to_ckpt = None):
-
-    ''' learning rate decay '''
-    def adjust_learning_rate_1(optimizer, epoch):
-        """decrease the learning rate """
-        lr = lr_base
-
-        num_decays = len(lr_decay_epochs)
-        for decay_i in range(num_decays):
-            if epoch >= lr_decay_epochs[decay_i]:
-                lr = lr * lr_decay_factor
-            #end if epoch
-        #end for decay_i
-        for param_group in optimizer.param_groups:
-            param_group['lr'] = lr
-
-    net = net.cuda()
-    criterion = nn.MSELoss()
-    optimizer = torch.optim.SGD(net.parameters(), lr = lr_base, momentum= 0.9, weight_decay=weight_decay)
-
-    # resume training; load checkpoint
-    if path_to_ckpt is not None and resume_epoch>0:
-        save_file = path_to_ckpt + "/embed_x2y_ckpt_in_train/embed_x2y_checkpoint_epoch_{}.pth".format(resume_epoch)
-        checkpoint = torch.load(save_file)
-        net.load_state_dict(checkpoint['net_state_dict'])
-        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
-        torch.set_rng_state(checkpoint['rng_state'])
-    #end if
-
-    start_tmp = timeit.default_timer()
-    for epoch in range(resume_epoch, epochs):
-        net.train()
-        train_loss = 0
-        adjust_learning_rate_1(optimizer, epoch)
-        for _, (batch_train_images, batch_train_labels) in enumerate(trainloader):
-
-            # batch_train_images = nn.functional.interpolate(batch_train_images, size = (299,299), scale_factor=None, mode='bilinear', align_corners=False)
-
-            batch_train_images = batch_train_images.type(torch.float).cuda()
-            batch_train_labels = batch_train_labels.type(torch.float).view(-1,1).cuda()
-
-            #Forward pass
-            outputs, _ = net(batch_train_images)
-            loss = criterion(outputs, batch_train_labels)
-
-            #backward pass
-            optimizer.zero_grad()
-            loss.backward()
-            optimizer.step()
-
-            train_loss += loss.cpu().item()
-        #end for batch_idx
-        train_loss = train_loss / len(trainloader)
-
-        if testloader is None:
-            print('Train net_x2y for embedding: [epoch %d/%d] train_loss:%f Time:%.4f' % (epoch+1, epochs, train_loss, timeit.default_timer()-start_tmp))
-        else:
-            net.eval()  # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance)
-            with torch.no_grad():
-                test_loss = 0
-                for batch_test_images, batch_test_labels in testloader:
-                    batch_test_images = batch_test_images.type(torch.float).cuda()
-                    batch_test_labels = batch_test_labels.type(torch.float).view(-1,1).cuda()
-                    outputs,_ = net(batch_test_images)
-                    loss = criterion(outputs, batch_test_labels)
-                    test_loss += loss.cpu().item()
-                test_loss = test_loss/len(testloader)
-
-                print('Train net_x2y for label embedding: [epoch %d/%d] train_loss:%f test_loss:%f Time:%.4f' % (epoch+1, epochs, train_loss, test_loss, timeit.default_timer()-start_tmp))
-
-        #save checkpoint
-        if path_to_ckpt is not None and (((epoch+1) % 50 == 0) or (epoch+1==epochs)):
-            save_file = path_to_ckpt + "/embed_x2y_ckpt_in_train/embed_x2y_checkpoint_epoch_{}.pth".format(epoch+1)
-            os.makedirs(os.path.dirname(save_file), exist_ok=True)
-            torch.save({
-                    'epoch': epoch,
-                    'net_state_dict': net.state_dict(),
-                    'optimizer_state_dict': optimizer.state_dict(),
-                    'rng_state': torch.get_rng_state()
-            }, save_file)
-    #end for epoch
-
-    return net
-
-
-
-
-###################################################################################
-class label_dataset(torch.utils.data.Dataset):
-    def __init__(self, labels):
-        super(label_dataset, self).__init__()
-
-        self.labels = labels
-        self.n_samples = len(self.labels)
-
-    def __getitem__(self, index):
-
-        y = self.labels[index]
-        return y
-
-    def __len__(self):
-        return self.n_samples
-
-
-def train_net_y2h(unique_labels_norm, net_y2h, net_embed, epochs=500, lr_base=0.01, lr_decay_factor=0.1, lr_decay_epochs=[150, 250, 350], weight_decay=1e-4, batch_size=128):
-    '''
-    unique_labels_norm: an array of normalized unique labels
-    '''
-
-    ''' learning rate decay '''
-    def adjust_learning_rate_2(optimizer, epoch):
-        """decrease the learning rate """
-        lr = lr_base
-
-        num_decays = len(lr_decay_epochs)
-        for decay_i in range(num_decays):
-            if epoch >= lr_decay_epochs[decay_i]:
-                lr = lr * lr_decay_factor
-            #end if epoch
-        #end for decay_i
-        for param_group in optimizer.param_groups:
-            param_group['lr'] = lr
-
-
-    assert np.max(unique_labels_norm)<=1 and np.min(unique_labels_norm)>=0
-    trainset = label_dataset(unique_labels_norm)
-    trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True)
-
-    net_embed.eval()
-    net_h2y=net_embed.module.h2y #convert embedding labels to original labels
-    optimizer_y2h = torch.optim.SGD(net_y2h.parameters(), lr = lr_base, momentum= 0.9, weight_decay=weight_decay)
-
-    start_tmp = timeit.default_timer()
-    for epoch in range(epochs):
-        net_y2h.train()
-        train_loss = 0
-        adjust_learning_rate_2(optimizer_y2h, epoch)
-        for _, batch_labels in enumerate(trainloader):
-
-            batch_labels = batch_labels.type(torch.float).view(-1,1).cuda()
-
-            # generate noises which will be added to labels
-            batch_size_curr = len(batch_labels)
-            batch_gamma = np.random.normal(0, 0.2, batch_size_curr)
-            batch_gamma = torch.from_numpy(batch_gamma).view(-1,1).type(torch.float).cuda()
-
-            # add noise to labels
-            batch_labels_noise = torch.clamp(batch_labels+batch_gamma, 0.0, 1.0)
-
-            #Forward pass
-            batch_hiddens_noise = net_y2h(batch_labels_noise)
-            batch_rec_labels_noise = net_h2y(batch_hiddens_noise)
-
-            loss = nn.MSELoss()(batch_rec_labels_noise, batch_labels_noise)
-
-            #backward pass
-            optimizer_y2h.zero_grad()
-            loss.backward()
-            optimizer_y2h.step()
-
-            train_loss += loss.cpu().item()
-        #end for batch_idx
-        train_loss = train_loss / len(trainloader)
-
-        print('\n Train net_y2h: [epoch %d/%d] train_loss:%f Time:%.4f' % (epoch+1, epochs, train_loss, timeit.default_timer()-start_tmp))
-    #end for epoch
-
-    return net_y2h