tools 1.0__py3-none-any.whl

tools/__init__.py

@@ -0,0 +1,14 @@
+
+__version__ = "1.0"
+from .tools import *
+from . import os
+
+'''
+For third-party packages, import manually by something like:
+import tools.pytorch
+import tools.pytorch as tpytorch
+import tools.pytorch as tpt
+import tools.pytorch as Tpt
+'''
+
+# TODO: make animator in plot/

tools/array.py

@@ -0,0 +1,39 @@
+import numpy as np
+import torch
+import tools as T
+
+
+# TODO: change name mangic_cat() -> concat()
+__all__ = [
+'magic_cat',
+]
+# %%
+def magic_cat(datas, axis=0, out=None):
+    '''concatenate numpy or torch tensors'''
+    assert T.equal([type(data) for data in datas]), 'Given data in datas must be the same type'
+    datatype = type(datas[0])
+    if datatype == torch.Tensor:
+        return torch.cat(datas, dim=axis, out=out)
+    elif datatype == np.ndarray or list:
+        return np.concatenate(datas, axis=axis, out=out)
+    else:
+        raise Exception(f'Elements in data must be either [list, np.ndarray, torch.Tensor], received: {datatype}')
+
+
+# Just use merge_dict and [value for value in variable]
+
+# def dict_np_cat(d_array, axis=0):
+#     '''
+#     concatenates list of numpy array and returns concatenated dict of numpy arrays
+#     '''
+#     d_cat = {}
+#     for key in d_array.keys():
+#         d[key] = d
+#     np.concatenate
+#     return d_cat
+#
+# def dict_torchcat(d_array, axis=0):
+#     pass
+#
+# def dict_magic_cat(d_array, axis=0):
+#     pass

tools/data.py

@@ -0,0 +1,90 @@
+import os
+import numpy as np
+from .tools import load_pickle
+
+def sort_load(data_dir, load_func = None):
+    '''load all data in the specified directory, in a sorted way
+    if load_func == None, defaults to pickle'''
+    if load_func == None:
+        load_func = load_pickle
+
+    file_list = sorted(os.listdir(data_dir))
+    loaded = list()
+
+    for file in file_list:
+        data_dir_1 = os.path.join(data_dir, file)
+        data = load_func(data_dir_1)
+        loaded.append(data)
+
+    return loaded
+
+def sample_train_data(dataset_A, dataset_B,ppgset_A,ppgset_B, n_frames=128):
+
+    num_samples = min(len(dataset_A), len(dataset_B))
+    train_data_A_idx = np.arange(len(dataset_A))
+    train_data_B_idx = np.arange(len(dataset_B))
+    np.random.shuffle(train_data_A_idx)
+    np.random.shuffle(train_data_B_idx)
+    train_data_A_idx_subset = train_data_A_idx[:num_samples]
+    train_data_B_idx_subset = train_data_B_idx[:num_samples]
+
+    train_data_ppg_A = list()
+    train_data_ppg_B = list()
+    train_data_A = list()
+    train_data_B = list()
+
+    for idx_A, idx_B in zip(train_data_A_idx_subset, train_data_B_idx_subset):
+        data_A = dataset_A[idx_A]
+        data_ppg_A = ppgset_A[idx_A]
+        frames_A_total = data_A.shape[1]
+        frames_A_ppg_total = data_ppg_A.shape[1]
+        #print(frames_A_total)
+        #print(frames_A_ppg_total)
+        #print(min([frames_A_total,frames_A_ppg_total]))
+        assert min([frames_A_total,frames_A_ppg_total]) >= n_frames
+        start_A = np.random.randint(min([frames_A_total,frames_A_ppg_total]) - n_frames + 1)
+        end_A = start_A + n_frames
+        train_data_A.append(data_A[:, start_A:end_A])
+        train_data_ppg_A.append(data_ppg_A[:, start_A:end_A])
+
+        data_B = dataset_B[idx_B]
+        data_ppg_B = ppgset_B[idx_B]
+        frames_B_total = data_ppg_B.shape[1]
+        frames_B_ppg_total = data_ppg_B.shape[1]
+        #print(min([frames_B_total,frames_B_ppg_total]))
+        assert min([frames_B_total,frames_B_ppg_total]) >= n_frames
+        start_B = np.random.randint(min([frames_B_total,frames_B_ppg_total]) - n_frames + 1)
+        end_B = start_B + n_frames
+        train_data_B.append(data_B[:, start_B:end_B])
+        train_data_ppg_B.append(data_ppg_B[:, start_B:end_B])
+        #print(np.shape(data_B))#data_B
+    #print(len(train_data_A))
+    #print(len(train_data_B))
+    train_data_ppg_A = np.array(train_data_ppg_A)
+    train_data_ppg_B = np.array(train_data_ppg_B)
+    train_data_A = np.array(train_data_A)
+    train_data_B = np.array(train_data_B)
+
+    #train_data_A = np.expand_dims(train_data_A, axis=-1)
+    #train_data_B = np.expand_dims(train_data_B, axis=-1)
+    return train_data_A, train_data_B,train_data_ppg_A,train_data_ppg_B
+
+class Tree():
+    '''Not implemented yet'''
+    def __init__(self, data = None, parent = None):
+        self.data = data
+
+        self.parent = parent
+        self.children = list()
+        self.depth = 0
+
+    def __call__(self):
+        return self.data
+
+    def __getitem__(self, key):
+        pass
+
+    def __setitem__(self, key):
+        pass
+    def __len__(self):
+        pass

tools/exp.py

@@ -0,0 +1,165 @@
+class Experiment(object):
+    '''
+    ------------------
+    Data descriptors:
+    dirs
+        Hold all directory information to be used in the experiment
+        in dict() form.
+        To see all directories that are related, refer to:
+        list(self.dirs.items())
+    model_p
+        Hold all parameters related to NN model learning in dict() form
+    train_p
+        Hold all parameters related to training in dict() form
+    speaker_list
+        Hold all name of speakers
+    '''
+    def __init__(self, num_speakers = 4, exp_name = None, exp_dir='exp', new = True, model_p = None, train_p = None, lambd = None, debug = False):
+        # 0] Random seed
+        np.random.seed(0)
+        torch.manual_seed(0)
+        torch.backends.cudnn.deterministic = True
+        torch.backends.cudnn.benchmark = False
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+        # 1] Hyperparameters setting - Default
+        self.model_p = dict(
+        vae_lr = 1e-2,
+        vae_betas = (0.9,0.999),
+        sc_lr = 0.0002,
+        sc_betas = (0.5,0.999),
+        asr_lr = 0.00001,
+        asr_betas = (0.5,0.999),
+        ac_lr = 0.00005,
+        ac_betas = (0.5,0.999),
+        )
+        if model_p is not None:
+            self.model_p.update(model_p)
+
+        self.train_p = dict(
+        n_train_frames = 128,
+        batch_size = 32,
+        mini_batch_size = 8,
+        start_epoch = 1,
+        n_epoch = 300,
+        model_save_epoch = 3,
+        validation_epoch = 3,
+        sample_per_path = 10,
+        )
+        self.train_p['iter_per_ep'] = self.train_p['batch_size'] // self.train_p['mini_batch_size']
+        if train_p is not None:
+            self.train_p.update(train_p)
+        try:
+            assert self.train_p['iter_per_ep'] * self.train_p['mini_batch_size'] == self.train_p['batch_size'], 'Specified batch_size "%s" cannot be divided by mini_batch_size "%s"'%(self.train_p['batch_size'], self.train_p['mini_batch_size'])
+        except:
+            print("Invalid train_p['iter_per_ep'] setting!")
+            print('iter_per_ep: %s'%self.train_p['iter_per_ep'])
+            print('batch_size: %s'%self.train_p['batch_size'])
+            print('mini_batch_size: %s'%self.train_p['mini_batch_size'])
+            print('Setting (iter_per_ep) = (batch_size) // (mini_batch_size)')
+            self.train_p['iter_per_ep'] = self.train_p['batch_size'] // self.train_p['mini_batch_size']
+        self.train_p['epoch'] = self.train_p['start_epoch'] - 1
+
+        self.lambd = dict(
+        KLD = 1,
+        rec = 20,
+        SI = 0,
+        LI = 0,
+        AC = 0,
+        SC = 0,
+        C = 0,
+        CC = 0,
+        )
+        if lambd is not None:
+            self.lambd.update(lambd)
+        self.lambd_total = sum(self.lambd.values())
+        if self.lambd['C'] is not 0:
+            self.lambd_total -= self.lambd['C']
+            self.lambd_total += self.lambd['C'] * (self.lambd['KLD'] + self.lambd['rec'])
+        self.lambda_norm = True
+
+        self.preprocess_p = dict(
+        sr = 16000,
+        frame_period = 5.0,
+        num_mcep = 36,
+        )
+        self.loss_index = ['loss_VAE','loss_KLD','loss_rec','loss_SI','loss_LI','loss_AC','loss_SC','loss_C_KLD', 'loss_C_rec']
+        self.performance_measure_index = ['mcd', 'msd_vector', 'gv']
+        self.lr_index = ['VAE_lr']
+        self.loss_summary = pd.DataFrame(columns = self.loss_index)
+        self.validation_summary = pd.DataFrame(columns = self.performance_measure_index)
+        self.lr_summary = pd.DataFrame(columns = self.lr_index)
+
+        self.model_kept = []
+        self.max_keep=100
+
+        # 2] Initialize environment and variables
+        self.create_env(exp_dir = exp_dir, exp_name = exp_name, new = new)
+        # self.speaker_list = sorted(os.listdir(self.dirs['train_data']))
+        self.speaker_list = ['p225','p226','p227','p228']
+        self.num_speakers = len(self.speaker_list)
+        assert self.num_speakers == num_speakers, 'Specified "num_speakers" and "num_speakers in train data" does not match'
+        self.build_model(params = self.model_p)
+        self.p = Printer(filewrite_dir = self.dirs['log'])
+        if debug == False:
+            sys.stdout = open(self.dirs['log_all'], 'a')
+        append(self.dirs['loss_log'], 'epoch '+' '.join(self.loss_index)+'\n')
+        append(self.dirs['validation_log'], 'epoch '+' '.join(self.performance_measure_index)+'\n')
+
+        # 3] Hyperparameters for saving model
+
+
+        # 4] If the experiment is not new, Load most recent model
+        if new == False:
+            self.model_kept= sorted(os.listdir(self.dirs['model']), key = lambda x: int(x.split('.')[0].split('_')[-1]))
+            most_trained_model = self.model_kept[-1]
+            epoch_trained = int(most_trained_model.split('_')[-1].split('.')[0])
+            self.train_p['start_epoch'] += epoch_trained
+            # Update lr_scheduler
+            print('Loading model from %s'%most_trained_model)
+            self.load_model_all(self.dirs['model'], epoch_trained)
+
+    def create_env(self, exp_dir = 'exp', exp_name = None, new = True):
+        '''Create experiment environment
+        Store all "static directories" required for experiment in "self.dirs"(dict)
+
+        Store every experiment result in: exp/exp_name/ == exp_dir
+        including log, model, test(validation) etc
+        '''
+        # 0] exp_dir == master directory
+        self.dirs = dict()
+        # exp_dir = 'exp/'
+        model_dir = 'model/'
+
+        # 1] Set up Experiment directory
+        if exp_name == None:
+            exp_name = time.strftime('%m%d_%H%M%S')
+        self.dirs['exp'] = os.path.join(exp_dir, exp_name)
+        if new == True:
+            assert not os.path.isdir(self.dirs['exp']), 'New experiment, but exp_dir with same name exists'
+            os.makedirs(self.dirs['exp'])
+        else:
+            assert os.path.isdir(self.dirs['exp']), 'Existing experiment, but exp_dir doesn\'t exist'
+
+        # 2] Model parameter directory
+
+    def save_log(self, result, log_dir):
+        # 1. Write to log
+        log_content = str(self.train_p['epoch'])
+        for value in result.mean():
+            log_content += ' '+str(value)
+        append(log_dir, log_content+'\n')
+        # 2. Print result statistics
+        self.p.print('Mean\n' + str(result.mean().to_frame().T))
+        self.p.print('Std\n' + str(result.std().to_frame().T))
+
+    def save_plot(self, summary, plot_dir):
+        for measure in summary.columns:
+            fig_save_dir = os.path.join(plot_dir, measure+'.png')
+            axes = summary.plot(y = measure, style='o-')
+            fig = axes.get_figure()
+            fig.savefig(fig_save_dir)
+        plt.close('all')
+
+    def performance_measure(self):
+        pass

tools/hydra/__init__.py

@@ -0,0 +1,6 @@
+import hydra
+from omegaconf import OmegaConf, DictConfig
+
+# %%
+def print_cfg(cfg: DictConfig) -> None:
+    print(OmegaConf.to_yaml(cfg))

tools/modules.py

@@ -0,0 +1,117 @@
+
+import matplotlib.pyplot as plt
+import numpy as np
+from . import numpy as tnp
+
+class ValueTracker(object):
+    """ ValueTracker."""
+
+    def __repr__(self):
+        return f'<ValueTracker>\nx: {self.x}\ny: {self.y}'
+
+    def __init__(self):
+        self.reset()
+
+    def __len__(self):
+        return len(self.y)
+
+    def __iadd__(self, other):
+        self.x.extend(other.x)
+        self.y.extend(other.y)
+        self.label.extend(other.label)
+        self.n_step += len(other.x)
+        return self
+
+    def __add__(self, other):
+        self = deepcopy(self)
+        self.x.extend(other.x)
+        self.y.extend(other.y)
+        self.label.extend(other.label)
+        self.n_step += len(other.x)
+        return self
+
+    def reset(self):
+        self.x = []
+        self.y = []
+        self.label = []
+        self.n_step = 0
+
+    def numpy(self):
+        return np.array(self.x), np.array(self.y), np.array(self.label)
+
+    def step(self, x, y, label=None):
+        if hasattr(x, '__len__'):
+            assert hasattr(y, '__len__')
+            assert len(x)==len(y)
+            self.x.extend(x)
+            self.y.extend(y)
+            if label != None:
+                assert len(y)==len(label)
+                self.label.extend(label)
+            self.n_step += len(x)
+
+        else:
+            self.x.append(x)
+            self.y.append(y)
+            if label != None:
+                self.label.append(label)
+            self.n_step += 1
+
+    def plot(self, w=9, color='tab:blue', ax=None):
+        x = np.array(self.x)
+        y = np.array(self.y)
+        y_smooth = tnp.moving_mean(y, w)
+        if ax==None:
+            ax = plt.gca()
+        ax.plot(x, y, color=color, alpha=0.4)
+        ax.plot(x, y_smooth, color=color)
+        return ax
+
+    def mean(self):
+        return np.mean(self.y)
+    def min(self):
+        return np.min(self.y)
+    def max(self):
+        return np.max(self.y)
+
+class AverageMeter(object):
+    """Computes and stores the average and current value
+    Variables
+    ---------
+    self.val
+    self.avg
+    self.sum
+    self.count
+    """
+    # TODO: maybe keep track of each values? or just merge with valuetracker?
+
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        self.val = 0
+        self.avg = 0
+        self.sum = 0
+        self.count = 0
+
+    def step(self, val, n=1):
+        self.val = val
+        self.sum += val * n
+        self.count += n
+        self.avg = self.sum / self.count
+
+class DictList(object):
+    """Dictionary of lists"""
+    def __init__(self, keys):
+        self._dict = {}
+
+    def append(self, data):
+        assert type(data) in [list, dict]
+        if type(data)==dict:
+            assert set(self._dict.keys())==set(data.keys()), f'allowed keys are: {self._dict.keys()}, received: {data.keys()}'
+            for key, value in data.items():
+                self._dict[key].append(item)
+        else:
+            warnings.warn('Appending with list is not recommended as this cannot ensure the data are being appended to the right place.')
+            for key, value in zip(self._dict.items(), data):
+                self._dict[key].append(value)

tools/numpy/__init__.py

@@ -0,0 +1,8 @@
+from ._f import *
+from ._utils import *
+
+if __name__ == '__main__':
+    ld = [{i:i+1 for i in range(5)} for j in range(3)]
+    merge_dict(ld)
+    ld = [{i:np.arange(i+1) for i in range(5)} for j in range(3)]
+    merge_dict(ld)

tools/numpy/_f.py

@@ -0,0 +1,60 @@
+import numpy as np
+
+# %%
+__all__ = [
+'angle',
+'binarize',
+'ceil',
+'floor',
+'moving_mean',
+'standardize',
+]
+
+# %%
+def floor(x, decimals=0):
+    if decimals==0:
+        return np.floor(x)
+    else:
+        return np.floor(x * 10**decimals) / 10**decimals
+
+def ceil(x, decimals=0):
+    if decimals==0:
+        return np.ceil(x)
+    else:
+        return np.ceil(x * 10**decimals) / 10**decimals
+
+def standardize(array, axis=None, ep=1e-20):
+    return (array - array.mean(axis=axis))/(array.std(axis=axis)+ep)
+
+def binarize(array, threshold):
+    '''binarize array with array>=threshold == 1'''
+    result = np.zeros_like(array)
+    result[array>=threshold] = 1
+    return result
+
+def moving_mean(x, w):
+    odd = bool(w%2)
+    edge = w//2
+    if odd:
+        x = np.pad(x, w//2+1, mode='edge')
+        x = np.cumsum(x).astype(np.float64)
+        x = (x[w:] - x[:-w])/w
+        return x[:-1]
+    else:
+        x = np.pad(x, w//2, mode='edge')
+        x = np.cumsum(x).astype(np.float64)
+        x = (x[w:] - x[:-w])/w
+        return x
+
+def angle(x1, x2):
+    '''angle between two vectors, derived from cosine rule
+    return theta within range of [0,np.pi]'''
+    theta = np.arccos(x1@x2/(np.linalg.norm(x1)*np.linalg.norm(x2)))
+    return theta
+
+def sigmoid(x):
+    return 1/(1+np.exp(-x))
+
+# %%
+if __name__ == '__main__':
+    pass

tools/numpy/_utils.py

@@ -0,0 +1,117 @@
+import numpy as np
+
+import tools as T
+
+# %%
+__all__ = [
+'equal',
+'merge_dict',
+'Squeezer'
+]
+
+# %%
+def equal(array, axis=None):
+    """
+    Check if all elements in the array are equal along the specified axis.
+
+    Parameters
+    ----------
+    array : array-like
+        Input array to check for equality.
+    axis : int, optional
+        Axis along which to check for equality. If None, check the entire array.
+
+    Returns
+    -------
+    bool or ndarray of bool
+        if axis is None, return True if all elements are equal, False otherwise.
+        if axis is specified, return an array of bools with axis dimension collapsed 
+    """
+    array = np.asarray(array)
+    if axis is None:
+        return np.all(array == array.flat[0])
+    else:
+        return np.all(array == np.expand_dims(array.take(0, axis=axis), axis=axis), axis=axis)
+
+def merge_dict(ld):
+    '''
+    :param ld: list of dicts [{}, {}, {}]
+    '''
+    assert type(ld) == list, f'must give list of dicts, received: {type(ld)}'
+    assert T.equal([list(d.keys()) for d in ld]), 'keys for every dict in the list of dicts must be the same'
+    keys = ld[0].keys()
+    merged_d = {}
+    for k in keys:
+        lv = [d[k] for d in ld] # list of values
+        try:
+            merged_d[k] = np.concatenate(lv, axis=0)
+        except ValueError:
+            merged_d[k] = np.array(lv)
+    return merged_d
+
+class Squeezer(object):
+    """
+    Warning: The class is very unstable, currently used as a temporary adjustment for concatenating / splitting batch dimension.
+
+    Reshape numpy arrays or lists of numpy arrays.
+    """
+    def __init__(self):
+        self.type = None
+        self.ndim = None
+        self.shape_original = None
+
+    def squeeze(self, x):
+        if isinstance(x, np.ndarray):
+            self.type = np.ndarray
+            self.ndim = x.ndim
+            if self.ndim==2:
+                self.shape_original = x.shape
+            elif x.ndim==3:
+                self.shape_original = x.shape
+                x.shape = (self.shape_original[0]*self.shape_original[1], self.shape_original[2])
+            else:
+                raise Exception(f'x.ndim must be 2 or 3, received: {x.ndim}')
+        elif isinstance(x, list):
+            self.type = list
+            if len(x) != 0:
+                assert isinstance(x[0], np.ndarray), 'values in list must be np.ndarray'
+                assert x[0].ndim==2, f'arrays in list must have ndim==2, received: {x[0].ndim}'
+                self.ndim = 3
+                self.shape_original = [x_.shape for x_ in x]
+                x = np.concatenate(x, axis=0)
+        else:
+            raise TypeError('Input must be a numpy array or a list of numpy arrays.')
+        
+        return x
+
+    def unsqueeze(self, x, strict=True):
+        if self.type is None or self.ndim is None or self.shape_original is None:
+            raise ValueError("Squeezer instance was not initialized correctly.")
+
+        if self.ndim == 3:
+            if self.type == np.ndarray:
+                if strict:
+                    if x.size == np.prod(self.shape_original):
+                        x.shape = self.shape_original
+                    else:
+                        raise ValueError(f'Input shape {x.shape} cannot be reshaped into {self.shape_original}.')
+                else:  # Allow flexibility in last dimension
+                    expected_elements = np.prod(self.shape_original[:-1])
+                    if x.size % expected_elements == 0:
+                        x.shape = (*self.shape_original[:-1], -1)
+                    else:
+                        raise ValueError(f'Cannot reshape {x.shape} flexibly; incorrect number of elements.')
+            elif self.type == list:
+                split_indices = np.cumsum([shape[0] for shape in self.shape_original[:-1]])
+                if x.shape[0] != sum(shape[0] for shape in self.shape_original):
+                    raise ValueError(f'Input shape {x.shape} does not match expected concatenated shape.')
+                x = np.split(x, split_indices, axis=0)
+
+        return x
+
+# %%
+if __name__ == '__main__':
+    ld = [{i:i+1 for i in range(5)} for j in range(3)]
+    merge_dict(ld)
+    ld = [{i:np.arange(i+1) for i in range(5)} for j in range(3)]
+    merge_dict(ld)

tools/os.py

@@ -0,0 +1,67 @@
+import os
+from typing import Union
+__all__ = \
+[
+'listdir',
+'makedirs'
+]
+
+def makedirs(path, exist_ok=True):
+    os.makedirs(path, exist_ok=exist_ok)
+
+def listdir(path=None, isdir: bool=False, isfile: Union[bool, str]=False, join: bool=False):
+    '''
+    :func listdir:
+
+    :param path:
+    :param isdir:
+    :param isfile:
+    to get files which start with ".", set isfile=''.
+    :param join:
+    '''
+    # Safety check
+    if type(isfile)==str:
+        ext = isfile.lower() # extension
+        isfile_str = True
+        isfile = True
+    else:
+        isfile_str=False
+    # assert type(isfile)==str or type(isfile)==bool, 'isfile can be either str or bool'
+    # _isfile = True if type(isfile)==str else isfile
+    assert not (isdir and isfile), 'only one of argument "isdir" and "isfile" can be True'
+    # if path == None:
+    #     path = '.'
+
+    dir_list = os.listdir(path)
+
+    # when path is referring to somewhere non-cwd, we need dir_list_joined to pass reference for isdir() or isfile().
+    if join or isdir or isfile:
+        dir_list_joined = dir_list if path is None else [os.path.join(path, dir) for dir in dir_list]
+
+    if join:
+        dir_list = dir_list_joined
+
+    if isdir:
+        return [dir for dir, dir_joined in zip(dir_list, dir_list_joined) if os.path.isdir(dir_joined)]
+    elif isfile:
+        if isfile_str:
+            return [dir for dir, dir_joined in zip(dir_list, dir_list_joined) if os.path.isfile(dir_joined) and (os.path.splitext(dir)[1].lower() == ext)]
+        else:
+            return [dir for dir, dir_joined in zip(dir_list, dir_list_joined) if os.path.isfile(dir_joined)]
+    else:
+        return dir_list
+
+# if __name__ == '__main__':
+#     listdir('torch', isdir=False, join=False)
+#     listdir('torch', isdir=False, join=True)
+#     listdir('torch', isdir=True, join=False)
+#     listdir('torch', isdir=True, join=True)
+#
+#     listdir('torch', isfile=True, join=False)
+#     listdir('torch', isfile=True, join=True)
+#     listdir('torch', isfile='.p', join=False)
+#     listdir('torch', isfile='.py', join=True)
+#     listdir('torch', isfile='', join=True)
+#
+#     listdir('torch', isdir=True, isfile=True)
+#     listdir('torch', isdir=True, isfile='py')