Myosotis-Researches 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl

myosotis_researches/CcGAN/train_128/pretrain_CNN_regre.py

@@ -1,255 +0,0 @@
-"""
-
-Pre-train a CNN on the whole dataset for evaluation purpose
-
-"""
-import os
-import argparse
-import shutil
-import timeit
-import torch
-import torchvision
-import torchvision.transforms as transforms
-import numpy as np
-import torch.nn as nn
-import torch.backends.cudnn as cudnn
-import random
-import matplotlib.pyplot as plt
-import matplotlib as mpl
-from torch import autograd
-from torchvision.utils import save_image
-import csv
-from tqdm import tqdm
-import gc
-import h5py
-
-
-#############################
-# Settings
-#############################
-
-parser = argparse.ArgumentParser(description='Pre-train CNNs')
-parser.add_argument('--root_path', type=str, default='')
-parser.add_argument('--data_path', type=str, default='')
-parser.add_argument('--num_workers', type=int, default=0)
-parser.add_argument('--CNN', type=str, default='ResNet34_regre',
-                    help='CNN for training; ResNetXX')
-parser.add_argument('--epochs', type=int, default=200, metavar='N',
-                    help='number of epochs to train CNNs (default: 200)')
-parser.add_argument('--batch_size_train', type=int, default=256, metavar='N',
-                    help='input batch size for training')
-parser.add_argument('--batch_size_valid', type=int, default=10, metavar='N',
-                    help='input batch size for testing')
-parser.add_argument('--base_lr', type=float, default=0.01,
-                    help='learning rate, default=0.1')
-parser.add_argument('--weight_dacay', type=float, default=1e-4,
-                    help='Weigth decay, default=1e-4')
-parser.add_argument('--seed', type=int, default=2020, metavar='S',
-                    help='random seed (default: 1)')
-parser.add_argument('--CVMode', action='store_true', default=False,
-                   help='CV mode?')
-parser.add_argument('--img_size', type=int, default=128, metavar='N')
-parser.add_argument('--min_label', type=float, default=0.0)
-parser.add_argument('--max_label', type=float, default=90.0)
-args = parser.parse_args()
-
-
-wd = args.root_path
-os.chdir(wd)
-from ..models_128 import *
-from .utils import IMGs_dataset
-
-# cuda
-device = torch.device("cuda")
-ngpu = torch.cuda.device_count()  # number of gpus
-
-# random seed
-random.seed(args.seed)
-torch.manual_seed(args.seed)
-torch.backends.cudnn.deterministic = True
-np.random.seed(args.seed)
-
-# directories for checkpoint, images and log files
-save_models_folder = wd + '/output/eval_models/'
-os.makedirs(save_models_folder, exist_ok=True)
-
-
-###########################################################################################################
-# Data
-###########################################################################################################
-# data loader
-data_filename = args.data_path + '/Ra_' + str(args.img_size) + 'x' + str(args.img_size) + '.h5'
-hf = h5py.File(data_filename, 'r')
-labels = hf['labels'][:]
-labels = labels.astype(float)
-images = hf['images'][:]
-hf.close()
-N_all = len(images)
-assert len(images) == len(labels)
-
-q1 = args.min_label
-q2 = args.max_label
-indx = np.where((labels>q1)*(labels<q2)==True)[0]
-labels = labels[indx]
-images = images[indx]
-assert len(labels)==len(images)
-
-
-# normalize to [0, 1]
-#min_label = np.min(labels)
-#labels += np.abs(min_label)
-#max_label = np.max(labels)
-#labels /= max_label
-labels /= args.max_label
-
-
-# define training and validation sets
-if args.CVMode:
-    #80% Training; 20% valdation
-    indx_all = np.arange(len(labels))
-    np.random.shuffle(indx_all)
-    indx_valid = indx_all[0:int(0.2*len(labels))]
-    indx_train = indx_all[int(0.2*len(labels)):]
-
-    trainset = IMGs_dataset(images[indx_train], labels[indx_train], normalize=True)
-    trainloader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size_train, shuffle=True, num_workers=args.num_workers)
-    validset = IMGs_dataset(images[indx_valid], labels[indx_valid], normalize=True)
-    validloader = torch.utils.data.DataLoader(validset, batch_size=args.batch_size_valid, shuffle=False, num_workers=args.num_workers)
-
-else:
-    trainset = IMGs_dataset(images, labels, normalize=True)
-    trainloader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size_train, shuffle=True, num_workers=args.num_workers)
-
-
-
-
-###########################################################################################################
-# Necessary functions
-###########################################################################################################
-
-#initialize CNNs
-def net_initialization(Pretrained_CNN_Name, ngpu = 1):
-    if Pretrained_CNN_Name == "ResNet18_regre":
-        net = ResNet18_regre_eval(ngpu = ngpu)
-    elif Pretrained_CNN_Name == "ResNet34_regre":
-        net = ResNet34_regre_eval(ngpu = ngpu)
-    elif Pretrained_CNN_Name == "ResNet50_regre":
-        net = ResNet50_regre_eval(ngpu = ngpu)
-    elif Pretrained_CNN_Name == "ResNet101_regre":
-        net = ResNet101_regre_eval(ngpu = ngpu)
-
-    net_name = 'PreCNNForEvalGANs_' + Pretrained_CNN_Name #get the net's name
-    net = net.to(device)
-
-    return net, net_name
-
-#adjust CNN learning rate
-def adjust_learning_rate(optimizer, epoch, BASE_LR_CNN):
-    lr = BASE_LR_CNN
-    # if epoch >= 35:
-    #     lr /= 10
-    # if epoch >= 70:
-    #     lr /= 10
-    if epoch >= 50:
-        lr /= 10
-    if epoch >= 120:
-        lr /= 10
-    for param_group in optimizer.param_groups:
-        param_group['lr'] = lr
-
-
-def train_CNN():
-
-    start_tmp = timeit.default_timer()
-    for epoch in range(args.epochs):
-        net.train()
-        train_loss = 0
-        adjust_learning_rate(optimizer, epoch, args.base_lr)
-        for batch_idx, (batch_train_images, batch_train_labels) in enumerate(trainloader):
-
-            # batch_train_images = nn.functional.interpolate(batch_train_images, size = (299,299), scale_factor=None, mode='bilinear', align_corners=False)
-
-            batch_train_images = batch_train_images.type(torch.float).cuda()
-            batch_train_labels = batch_train_labels.type(torch.float).view(-1,1).cuda()
-
-            #Forward pass
-            outputs,_ = net(batch_train_images)
-            loss = criterion(outputs, batch_train_labels)
-
-            #backward pass
-            optimizer.zero_grad()
-            loss.backward()
-            optimizer.step()
-
-            train_loss += loss.cpu().item()
-        #end for batch_idx
-        train_loss = train_loss / len(trainloader)
-
-        if args.CVMode:
-            valid_loss = valid_CNN(verbose=False)
-            print('CNN: [epoch %d/%d] train_loss:%f valid_loss (avg_abs):%f Time: %.4f' % (epoch+1, args.epochs, train_loss, valid_loss, timeit.default_timer()-start_tmp))
-        else:
-            print('CNN: [epoch %d/%d] train_loss:%f Time:%.4f' % (epoch+1, args.epochs, train_loss, timeit.default_timer()-start_tmp))
-    #end for epoch
-
-    return net, optimizer
-
-if args.CVMode:
-    def valid_CNN(verbose=True):
-        net.eval()  # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance)
-        with torch.no_grad():
-            abs_diff_avg = 0
-            total = 0
-            for batch_idx, (images, labels) in enumerate(validloader):
-                images = images.type(torch.float).cuda()
-                labels = labels.type(torch.float).view(-1).cpu().numpy()
-                outputs,_ = net(images)
-                outputs = outputs.view(-1).cpu().numpy()
-                labels = labels * args.max_label
-                outputs = outputs * args.max_label
-                abs_diff_avg += np.sum(np.abs(labels-outputs))
-                total += len(labels)
-
-            if verbose:
-                print('Validation Average Absolute Difference: {}'.format(abs_diff_avg/total))
-        return abs_diff_avg/total
-
-
-
-###########################################################################################################
-# Training and validation
-###########################################################################################################
-
-
-# model initialization
-net, net_name = net_initialization(args.CNN, ngpu = ngpu)
-criterion = nn.MSELoss()
-optimizer = torch.optim.SGD(net.parameters(), lr = args.base_lr, momentum= 0.9, weight_decay=args.weight_dacay)
-
-filename_ckpt = save_models_folder + '/ckpt_{}_epoch_{}_seed_{}_CVMode_{}.pth'.format(net_name, args.epochs, args.seed, args.CVMode)
-
-
-# training
-if not os.path.isfile(filename_ckpt):
-    # TRAIN CNN
-    print("\n Begin training CNN: ")
-    start = timeit.default_timer()
-    net, optimizer = train_CNN()
-    stop = timeit.default_timer()
-    print("Time elapses: {}s".format(stop - start))
-    # save model
-    torch.save({
-    'net_state_dict': net.state_dict(),
-    }, filename_ckpt)
-else:
-    print("\n Ckpt already exists")
-    print("\n Loading...")
-    checkpoint = torch.load(filename_ckpt)
-    net.load_state_dict(checkpoint['net_state_dict'])
-torch.cuda.empty_cache()#release GPU mem which is  not references
-
-
-if args.CVMode:
-    #validation
-    _ = valid_CNN(True)
-    torch.cuda.empty_cache()

myosotis_researches/CcGAN/train_128/train_ccgan.py

@@ -1,303 +0,0 @@
-import torch
-import numpy as np
-import os
-import timeit
-from PIL import Image
-from torchvision.utils import save_image
-import torch.cuda as cutorch
-
-from .utils import SimpleProgressBar, IMGs_dataset
-from .opts import parse_opts
-from .DiffAugment_pytorch import DiffAugment
-
-''' Settings '''
-args = parse_opts()
-
-# some parameters in opts
-gan_arch = args.GAN_arch
-loss_type = args.loss_type_gan
-niters = args.niters_gan
-resume_niters = args.resume_niters_gan
-dim_gan = args.dim_gan
-lr_g = args.lr_g_gan
-lr_d = args.lr_d_gan
-save_niters_freq = args.save_niters_freq
-batch_size_disc = args.batch_size_disc
-batch_size_gene = args.batch_size_gene
-# batch_size_max = max(batch_size_disc, batch_size_gene)
-num_D_steps = args.num_D_steps
-
-visualize_freq = args.visualize_freq
-
-num_workers = args.num_workers
-
-threshold_type = args.threshold_type
-nonzero_soft_weight_threshold = args.nonzero_soft_weight_threshold
-
-num_channels = args.num_channels
-img_size = args.img_size
-max_label = args.max_label
-
-use_DiffAugment = args.gan_DiffAugment
-policy = args.gan_DiffAugment_policy
-
-
-## normalize images
-def normalize_images(batch_images):
-    batch_images = batch_images/255.0
-    batch_images = (batch_images - 0.5)/0.5
-    return batch_images
-
-
-def train_ccgan(kernel_sigma, kappa, train_images, train_labels, netG, netD, net_y2h, save_images_folder, save_models_folder = None, clip_label=False):
-    
-    '''
-    Note that train_images are not normalized to [-1,1]
-    '''
-
-    netG = netG.cuda()
-    netD = netD.cuda()
-    net_y2h = net_y2h.cuda()
-    net_y2h.eval()
-
-    optimizerG = torch.optim.Adam(netG.parameters(), lr=lr_g, betas=(0.5, 0.999))
-    optimizerD = torch.optim.Adam(netD.parameters(), lr=lr_d, betas=(0.5, 0.999))
-
-    if save_models_folder is not None and resume_niters>0:
-        save_file = save_models_folder + "/CcGAN_{}_{}_nDsteps_{}_checkpoint_intrain/CcGAN_checkpoint_niters_{}.pth".format(gan_arch, threshold_type, num_D_steps, resume_niters)
-        checkpoint = torch.load(save_file)
-        netG.load_state_dict(checkpoint['netG_state_dict'])
-        netD.load_state_dict(checkpoint['netD_state_dict'])
-        optimizerG.load_state_dict(checkpoint['optimizerG_state_dict'])
-        optimizerD.load_state_dict(checkpoint['optimizerD_state_dict'])
-        torch.set_rng_state(checkpoint['rng_state'])
-    #end if
-
-    #################
-    unique_train_labels = np.sort(np.array(list(set(train_labels))))
-
-    # printed images with labels between the 5-th quantile and 95-th quantile of training labels
-    n_row=10; n_col = n_row
-    z_fixed = torch.randn(n_row*n_col, dim_gan, dtype=torch.float).cuda()
-    start_label = np.quantile(train_labels, 0.05)
-    end_label = np.quantile(train_labels, 0.95)
-    selected_labels = np.linspace(start_label, end_label, num=n_row)
-    y_fixed = np.zeros(n_row*n_col)
-    for i in range(n_row):
-        curr_label = selected_labels[i]
-        for j in range(n_col):
-            y_fixed[i*n_col+j] = curr_label
-    print(y_fixed)
-    y_fixed = torch.from_numpy(y_fixed).type(torch.float).view(-1,1).cuda()
-
-
-    start_time = timeit.default_timer()
-    for niter in range(resume_niters, niters):
-
-        '''  Train Discriminator   '''
-        for _ in range(num_D_steps):
-
-            ## randomly draw batch_size_disc y's from unique_train_labels
-            batch_target_labels_in_dataset = np.random.choice(unique_train_labels, size=batch_size_disc, replace=True)
-            ## add Gaussian noise; we estimate image distribution conditional on these labels
-            batch_epsilons = np.random.normal(0, kernel_sigma, batch_size_disc)
-            batch_target_labels = batch_target_labels_in_dataset + batch_epsilons
-
-            ## find index of real images with labels in the vicinity of batch_target_labels
-            ## generate labels for fake image generation; these labels are also in the vicinity of batch_target_labels
-            batch_real_indx = np.zeros(batch_size_disc, dtype=int) #index of images in the datata; the labels of these images are in the vicinity
-            batch_fake_labels = np.zeros(batch_size_disc)
-
-            for j in range(batch_size_disc):
-                ## index for real images
-                if threshold_type == "hard":
-                    indx_real_in_vicinity = np.where(np.abs(train_labels-batch_target_labels[j])<= kappa)[0]
-                else:
-                    # reverse the weight function for SVDL
-                    indx_real_in_vicinity = np.where((train_labels-batch_target_labels[j])**2 <= -np.log(nonzero_soft_weight_threshold)/kappa)[0]
-
-                ## if the max gap between two consecutive ordered unique labels is large, it is possible that len(indx_real_in_vicinity)<1
-                while len(indx_real_in_vicinity)<1:
-                    batch_epsilons_j = np.random.normal(0, kernel_sigma, 1)
-                    batch_target_labels[j] = batch_target_labels_in_dataset[j] + batch_epsilons_j
-                    if clip_label:
-                        batch_target_labels = np.clip(batch_target_labels, 0.0, 1.0)
-                    ## index for real images
-                    if threshold_type == "hard":
-                        indx_real_in_vicinity = np.where(np.abs(train_labels-batch_target_labels[j])<= kappa)[0]
-                    else:
-                        # reverse the weight function for SVDL
-                        indx_real_in_vicinity = np.where((train_labels-batch_target_labels[j])**2 <= -np.log(nonzero_soft_weight_threshold)/kappa)[0]
-                #end while len(indx_real_in_vicinity)<1
-
-                assert len(indx_real_in_vicinity)>=1
-
-                batch_real_indx[j] = np.random.choice(indx_real_in_vicinity, size=1)[0]
-
-                ## labels for fake images generation
-                if threshold_type == "hard":
-                    lb = batch_target_labels[j] - kappa
-                    ub = batch_target_labels[j] + kappa
-                else:
-                    lb = batch_target_labels[j] - np.sqrt(-np.log(nonzero_soft_weight_threshold)/kappa)
-                    ub = batch_target_labels[j] + np.sqrt(-np.log(nonzero_soft_weight_threshold)/kappa)
-                lb = max(0.0, lb); ub = min(ub, 1.0)
-                assert lb<=ub
-                assert lb>=0 and ub>=0
-                assert lb<=1 and ub<=1
-                batch_fake_labels[j] = np.random.uniform(lb, ub, size=1)[0]
-            #end for j
-
-            ## draw real image/label batch from the training set
-            batch_real_images = torch.from_numpy(normalize_images(train_images[batch_real_indx]))
-            batch_real_images = batch_real_images.type(torch.float).cuda()
-            batch_real_labels = train_labels[batch_real_indx]
-            batch_real_labels = torch.from_numpy(batch_real_labels).type(torch.float).cuda()
-
-
-            ## generate the fake image batch
-            batch_fake_labels = torch.from_numpy(batch_fake_labels).type(torch.float).cuda()
-            z = torch.randn(batch_size_disc, dim_gan, dtype=torch.float).cuda()
-            batch_fake_images = netG(z, net_y2h(batch_fake_labels))
-
-            ## target labels on gpu
-            batch_target_labels = torch.from_numpy(batch_target_labels).type(torch.float).cuda()
-
-            ## weight vector
-            if threshold_type == "soft":
-                real_weights = torch.exp(-kappa*(batch_real_labels-batch_target_labels)**2).cuda()
-                fake_weights = torch.exp(-kappa*(batch_fake_labels-batch_target_labels)**2).cuda()
-            else:
-                real_weights = torch.ones(batch_size_disc, dtype=torch.float).cuda()
-                fake_weights = torch.ones(batch_size_disc, dtype=torch.float).cuda()
-            #end if threshold type
-
-            # forward pass
-            if use_DiffAugment:
-                real_dis_out = netD(DiffAugment(batch_real_images, policy=policy), net_y2h(batch_target_labels))
-                fake_dis_out = netD(DiffAugment(batch_fake_images.detach(), policy=policy), net_y2h(batch_target_labels))
-            else:
-                real_dis_out = netD(batch_real_images, net_y2h(batch_target_labels))
-                fake_dis_out = netD(batch_fake_images.detach(), net_y2h(batch_target_labels))
-
-            if loss_type == "vanilla":
-                real_dis_out = torch.nn.Sigmoid()(real_dis_out)
-                fake_dis_out = torch.nn.Sigmoid()(fake_dis_out)
-                d_loss_real = - torch.log(real_dis_out+1e-20)
-                d_loss_fake = - torch.log(1-fake_dis_out+1e-20)
-            elif loss_type == "hinge":
-                d_loss_real = torch.nn.ReLU()(1.0 - real_dis_out)
-                d_loss_fake = torch.nn.ReLU()(1.0 + fake_dis_out)
-            else:
-                raise ValueError('Not supported loss type!!!')
-
-            d_loss = torch.mean(real_weights.view(-1) * d_loss_real.view(-1)) + torch.mean(fake_weights.view(-1) * d_loss_fake.view(-1))
-
-            optimizerD.zero_grad()
-            d_loss.backward()
-            optimizerD.step()
-
-        #end for step_D_index
-
-
-
-        '''  Train Generator   '''
-        netG.train()
-
-        # generate fake images
-        ## randomly draw batch_size_gene y's from unique_train_labels
-        batch_target_labels_in_dataset = np.random.choice(unique_train_labels, size=batch_size_gene, replace=True)
-        ## add Gaussian noise; we estimate image distribution conditional on these labels
-        batch_epsilons = np.random.normal(0, kernel_sigma, batch_size_gene)
-        batch_target_labels = batch_target_labels_in_dataset + batch_epsilons
-        batch_target_labels = torch.from_numpy(batch_target_labels).type(torch.float).cuda()
-
-        z = torch.randn(batch_size_gene, dim_gan, dtype=torch.float).cuda()
-        batch_fake_images = netG(z, net_y2h(batch_target_labels))
-
-        # loss
-        if use_DiffAugment:
-            dis_out = netD(DiffAugment(batch_fake_images, policy=policy), net_y2h(batch_target_labels))
-        else:
-            dis_out = netD(batch_fake_images, net_y2h(batch_target_labels))
-        if loss_type == "vanilla":
-            dis_out = torch.nn.Sigmoid()(dis_out)
-            g_loss = - torch.mean(torch.log(dis_out+1e-20))
-        elif loss_type == "hinge":
-            g_loss = - dis_out.mean()
-
-        # backward
-        optimizerG.zero_grad()
-        g_loss.backward()
-        optimizerG.step()
-
-        # print loss
-        if (niter+1) % 20 == 0:
-            print ("CcGAN,%s: [Iter %d/%d] [D loss: %.4e] [G loss: %.4e] [real prob: %.3f] [fake prob: %.3f] [Time: %.4f]" % (gan_arch, niter+1, niters, d_loss.item(), g_loss.item(), real_dis_out.mean().item(), fake_dis_out.mean().item(), timeit.default_timer()-start_time))
-
-        if (niter+1) % visualize_freq == 0:
-            netG.eval()
-            with torch.no_grad():
-                gen_imgs = netG(z_fixed, net_y2h(y_fixed))
-                gen_imgs = gen_imgs.detach().cpu()
-                save_image(gen_imgs.data, save_images_folder + '/{}.png'.format(niter+1), nrow=n_row, normalize=True)
-
-        if save_models_folder is not None and ((niter+1) % save_niters_freq == 0 or (niter+1) == niters):
-            save_file = save_models_folder + "/CcGAN_{}_{}_nDsteps_{}_checkpoint_intrain/CcGAN_checkpoint_niters_{}.pth".format(gan_arch, threshold_type, num_D_steps, niter+1)
-            os.makedirs(os.path.dirname(save_file), exist_ok=True)
-            torch.save({
-                    'netG_state_dict': netG.state_dict(),
-                    'netD_state_dict': netD.state_dict(),
-                    'optimizerG_state_dict': optimizerG.state_dict(),
-                    'optimizerD_state_dict': optimizerD.state_dict(),
-                    'rng_state': torch.get_rng_state()
-            }, save_file)
-    #end for niter
-    return netG, netD
-
-
-def sample_ccgan_given_labels(netG, net_y2h, labels, batch_size = 500, to_numpy=True, denorm=True, verbose=True):
-    '''
-    netG: pretrained generator network
-    labels: float. normalized labels.
-    '''
-
-    nfake = len(labels)
-    if batch_size>nfake:
-        batch_size=nfake
-
-    fake_images = []
-    fake_labels = np.concatenate((labels, labels[0:batch_size]))
-    netG=netG.cuda()
-    netG.eval()
-    net_y2h = net_y2h.cuda()
-    net_y2h.eval()
-    with torch.no_grad():
-        if verbose:
-            pb = SimpleProgressBar()
-        n_img_got = 0
-        while n_img_got < nfake:
-            z = torch.randn(batch_size, dim_gan, dtype=torch.float).cuda()
-            y = torch.from_numpy(fake_labels[n_img_got:(n_img_got+batch_size)]).type(torch.float).view(-1,1).cuda()
-            batch_fake_images = netG(z, net_y2h(y))
-            if denorm: #denorm imgs to save memory
-                assert batch_fake_images.max().item()<=1.0 and batch_fake_images.min().item()>=-1.0
-                batch_fake_images = batch_fake_images*0.5+0.5
-                batch_fake_images = batch_fake_images*255.0
-                batch_fake_images = batch_fake_images.type(torch.uint8)
-                # assert batch_fake_images.max().item()>1
-            fake_images.append(batch_fake_images.cpu())
-            n_img_got += batch_size
-            if verbose:
-                pb.update(min(float(n_img_got)/nfake, 1)*100)
-        ##end while
-
-    fake_images = torch.cat(fake_images, dim=0)
-    #remove extra entries
-    fake_images = fake_images[0:nfake]
-    fake_labels = fake_labels[0:nfake]
-
-    if to_numpy:
-        fake_images = fake_images.numpy()
-
-    return fake_images, fake_labels

myosotis_researches/CcGAN/train_128/utils.py

@@ -1,120 +0,0 @@
-"""
-Some helpful functions
-
-"""
-import numpy as np
-import torch
-import torch.nn as nn
-import torchvision
-import matplotlib.pyplot as plt
-import matplotlib as mpl
-from torch.nn import functional as F
-import sys
-import PIL
-from PIL import Image
-
-
-
-# ################################################################################
-# Progress Bar
-class SimpleProgressBar():
-    def __init__(self, width=50):
-        self.last_x = -1
-        self.width = width
-
-    def update(self, x):
-        assert 0 <= x <= 100 # `x`: progress in percent ( between 0 and 100)
-        if self.last_x == int(x): return
-        self.last_x = int(x)
-        pointer = int(self.width * (x / 100.0))
-        sys.stdout.write( '\r%d%% [%s]' % (int(x), '#' * pointer + '.' * (self.width - pointer)))
-        sys.stdout.flush()
-        if x == 100:
-            print('')
-
-
-
-################################################################################
-# torch dataset from numpy array
-class IMGs_dataset(torch.utils.data.Dataset):
-    def __init__(self, images, labels=None, normalize=False):
-        super(IMGs_dataset, self).__init__()
-
-        self.images = images
-        self.n_images = len(self.images)
-        self.labels = labels
-        if labels is not None:
-            if len(self.images) != len(self.labels):
-                raise Exception('images (' +  str(len(self.images)) +') and labels ('+str(len(self.labels))+') do not have the same length!!!')
-        self.normalize = normalize
-
-
-    def __getitem__(self, index):
-
-        image = self.images[index]
-
-        if self.normalize:
-            image = image/255.0
-            image = (image-0.5)/0.5
-
-        if self.labels is not None:
-            label = self.labels[index]
-            return (image, label)
-        else:
-            return image
-
-    def __len__(self):
-        return self.n_images
-
-
-def PlotLoss(loss, filename):
-    x_axis = np.arange(start = 1, stop = len(loss)+1)
-    plt.switch_backend('agg')
-    mpl.style.use('seaborn')
-    fig = plt.figure()
-    ax = plt.subplot(111)
-    ax.plot(x_axis, np.array(loss))
-    plt.xlabel("epoch")
-    plt.ylabel("training loss")
-    plt.legend()
-    #ax.legend(loc='upper center', bbox_to_anchor=(0.5, 1.15),  shadow=True, ncol=3)
-    #plt.title('Training Loss')
-    plt.savefig(filename)
-
-
-# compute entropy of class labels; labels is a numpy array
-def compute_entropy(labels, base=None):
-    value,counts = np.unique(labels, return_counts=True)
-    norm_counts = counts / counts.sum()
-    base = np.e if base is None else base
-    return -(norm_counts * np.log(norm_counts)/np.log(base)).sum()
-
-def predict_class_labels(net, images, batch_size=500, verbose=False, num_workers=0):
-    net = net.cuda()
-    net.eval()
-
-    n = len(images)
-    if batch_size>n:
-        batch_size=n
-    dataset_pred = IMGs_dataset(images, normalize=False)
-    dataloader_pred = torch.utils.data.DataLoader(dataset_pred, batch_size=batch_size, shuffle=False, num_workers=num_workers)
-
-    class_labels_pred = np.zeros(n+batch_size)
-    with torch.no_grad():
-        nimgs_got = 0
-        if verbose:
-            pb = SimpleProgressBar()
-        for batch_idx, batch_images in enumerate(dataloader_pred):
-            batch_images = batch_images.type(torch.float).cuda()
-            batch_size_curr = len(batch_images)
-
-            outputs,_ = net(batch_images)
-            _, batch_class_labels_pred = torch.max(outputs.data, 1)
-            class_labels_pred[nimgs_got:(nimgs_got+batch_size_curr)] = batch_class_labels_pred.detach().cpu().numpy().reshape(-1)
-
-            nimgs_got += batch_size_curr
-            if verbose:
-                pb.update((float(nimgs_got)/n)*100)
-        #end for batch_idx
-    class_labels_pred = class_labels_pred[0:n]
-    return class_labels_pred