tools 1.0__py3-none-any.whl

tools/pandas/__init__.py

@@ -0,0 +1,45 @@
+import numbers
+
+import numpy as np
+import pandas as pd
+
+def isclose(a, b, rtol=1e-09, atol=0.0, equal_nan=False):
+    """
+    Wraparound np.isclose for pandas.DataFrame
+
+    Parameters
+    ----------
+    a, b : pandas.DataFrame
+        Input arrays to compare.
+    rtol : float
+        The relative tolerance parameter (see Notes).
+    atol : float
+        The absolute tolerance parameter (see Notes).
+    equal_nan : bool
+        Whether to compare NaN's as equal.  If True, NaN's in `a` will be
+        considered equal to NaN's in `b` in the output array.
+
+    Returns
+    -------
+    C : pandas.DataFrame 
+        Returns a boolean array of where `a` and `b` are equal within the
+        given tolerance. If both `a` and `b` are scalars, returns a single
+        boolean value.
+    """
+    assert a.shape == b.shape, "Shape mismatch"
+    assert (a.columns == b.columns).all(), "Column mismatch"
+    assert (a.index == b.index).all(), "Index mismatch"
+
+    close_list = []
+    for col in a.columns:
+        if issubclass(a[col].dtype.type, numbers.Number):
+            close = np.isclose(a[col], b[col], rtol=rtol, atol=atol, equal_nan=equal_nan)
+        else:
+            close = (a[col] == b[col]).to_numpy()
+        close_list.append(close)
+    close = np.stack(close_list, axis=1)
+
+    return pd.DataFrame(close, columns=a.columns, index=a.index)
+
+if __name__ == '__main__':
+    pass

tools/plot/__init__.py

@@ -0,0 +1,5 @@
+from .sklearn import *
+from .utils import *
+
+if __name__ == '__main__':
+    pass

tools/plot/sklearn.py

@@ -0,0 +1,28 @@
+import sklearn.metrics as metrics
+import seaborn as sns
+import pandas as pd
+import matplotlib.pyplot as plt
+
+__all__ = [
+'confusion_matrix',
+'plot_confusion_matrix',
+]
+
+def plot_confusion_matrix(y_true, y_pred, ax=None, **kwargs):
+    '''Just a wrapper to generate the confusion matrix and the plot'''
+    c_matrix = metrics.confusion_matrix(y_true, y_pred)
+    return confusion_matrix(c_matrix, ax=ax)
+
+def confusion_matrix(c_matrix, ax=None, **kwargs):
+    '''Plot confusion matrix heatmap'''
+    df_cm = pd.DataFrame(c_matrix)
+    if ax is None:
+        ax = sns.heatmap(df_cm, annot=True, **kwargs)
+    else:
+        ax = sns.heatmap(df_cm, annot=True, ax=ax, **kwargs)
+    ax.set_ylabel('y_true')
+    ax.set_xlabel('y_pred')
+    return ax
+
+if __name__ == '__main__':
+    pass

tools/plot/utils.py

@@ -0,0 +1,70 @@
+# import logging
+import os
+import warnings
+
+import numpy as np
+import matplotlib.colors as mcolors
+from .. import numpy as tnp
+# log = logging.getLogger(__name__)
+
+__all__ = [
+'get_xtick_seconds',
+'multisave',
+]
+
+# %%
+def get_xtick_seconds(ax, sr):
+    '''
+    :param ax: axis instance
+    :param sec: second which maximum length indicates. it is assumed the xtick starts at 0
+    '''
+    # xticks = np.linspace(0,sec,len(ax_real.get_xticklabels())-2)
+    # xticks = np.round(xticks, 3)
+    # d = xticks[1]-xticks[0]
+    # xticks = np.concatenate([[xticks[0] - d],xticks,[xticks[-1]+d]])
+    # return xticks
+    return ax.get_xticks() / sr
+
+def multisave(fig, path, dpi=300):
+    '''
+    save given figure in .png, .eps, .svg format
+    :param fig: pyplot figure instance
+    :param path: path of the figure without extension (.extension)
+    '''
+    if '.' in os.path.basename(path):
+        warnings.warn("Do not specify '.' in name. assuming it's part of the filename...")
+
+        # log.warning('Do not specify extension in multisave(). Removing extension...')
+        # path = os.path.splitext(path)[0]
+
+    fig.savefig(path+'.png', dpi=dpi)
+    fig.savefig(path+'.tiff', dpi=dpi)
+    fig.savefig(path+'.eps')
+    fig.savefig(path+'.svg')
+
+def plot_values(valuetracker_list, ax=None):
+    n_line = len(valuetracker_list)
+    if ax is None:
+        fig, ax = plt.subplots()
+    color_list = it.cycle(mcolors.TABLEAU_COLORS)
+    for valuetracker, color in zip(valuetracker_list, color_list):
+        y_smooth = tnp.moving_mean(valuetracker.y, 9)
+        ax.plot(valuetracker.x, valuetracker.y, color=color, alpha=0.4)
+        ax.plot(valuetracker.x, y_smooth, color=color)
+    return ax
+
+def plot_trainval(valuetracker_list, ax=None):
+    n_line = len(valuetracker_list)
+    if ax is None:
+        fig, ax = plt.subplots()
+    color_list = ['tab:blue', 'tab:orange']
+    line_list = ['-', 'x-']
+    labels = ['train', 'val']
+    for valuetracker, color, line, label in zip(valuetracker_list, color_list, line_list, labels):
+        y_smooth = tnp.moving_mean(valuetracker.y, 9)
+        ax.plot(valuetracker.x, valuetracker.y, color=color, alpha=0.4, label=label)
+        ax.plot(valuetracker.x, y_smooth, line, color=color, label=label+'_smooth')
+    return ax
+
+if __name__ == '__main__':
+    pass

tools/random.py

@@ -0,0 +1,35 @@
+import random
+
+import numpy as np
+import torch
+
+# %%
+def choice(a, size=None, replace=True):
+    '''
+    Randomly choose elements from given iterable.
+
+    Parameters
+    ----------
+    a: int or list
+        choice of
+
+    '''
+    m = len(a)
+    # Single sample
+    if size==None:
+        random_i = np.random.randint(m)
+        return a[random_i]
+    else:
+        num_sample = np.prod(size)
+
+    # Multi-Sample
+    a=np.asarray(a)
+    if replace==True:
+        random_i = np.random.randint(m, size = num_sample)
+        return a[random_i].reshape(size)
+    else:
+        assert m >= num_sample, 'entries of array cannot exceed number of samples'
+        random_i = np.arange(m)
+        np.random.shuffle(random_i)
+        random_i = random_i[:num_sample]
+        return a[random_i].reshape(size)

tools/sklearn/__init__.py

@@ -0,0 +1,2 @@
+
+from . import model_selection, preprocessing

tools/sklearn/metrics.py

@@ -0,0 +1,82 @@
+from collections.abc import Iterable
+import warnings
+
+import numpy as np
+from sklearn.metrics import r2_score as r2_score_sklearn
+
+def squared_error(y_true, y_pred, axis=None):
+    '''MSE without mean
+    if axis is provided, it will return the mean along the axis
+    '''
+    score = (y_true-y_pred)**2
+    if axis is not None:
+        score = np.mean(score, axis=axis)
+    return score
+
+def absolute_error(y_true, y_pred, axis=None):
+    '''MAE without mean'''
+    score = np.abs(y_true-y_pred)
+    if axis is not None:
+        score = np.mean(score, axis=axis)
+    return score
+
+def r2_score(y_true, y_pred, axis=None, multioutput='raw_values'):
+    """
+    R^2 score for multidimensional predictions.
+    collapses all axes except the specified axis.
+
+    Parameters
+    ----------
+    y_true : np.ndarray
+    y_pred : np.ndarray
+    axis: int or iterable of int, default=None
+        Axis to collapse.
+        It must be specified if y_true and y_pred dimensions are > 2
+    multioutput : Reference to `sklearn.metrics.r2_score`
+        https://scikit-learn.org/stable/modules/generated/sklearn.metrics.r2_score.html
+    
+    Returns
+    -------
+    z : np.ndarray
+        if axis is specified, returns an array of shape (y_true.shape[axis],)
+    """
+    assert y_true.shape == y_pred.shape, f"y_true and y_pred must have the same shape, received: {y_true.shape} and {y_pred.shape}"
+    if axis is not None:
+        if not isinstance(axis, Iterable):
+            axis = [axis]
+        else:
+            axis = list(axis)
+        if len(axis) > y_true.ndim:
+            raise ValueError("Axis is greater than the number of dimensions of y_true and y_pred")
+
+        # shape_final = np.array(y_true.shape)[axis]
+        # shape_collapse = list(set(range(y_true.ndim)).difference(axis))
+        dim_collapse = axis
+        dim_final = list(set(range(y_true.ndim)).difference(axis))
+        shape_final = np.array(y_true.shape)[dim_final]
+
+        y_true = np.transpose(y_true, (*dim_collapse, *dim_final)).reshape(-1, np.prod(shape_final)) # Move axis to the end, and flatten the rest
+        y_pred = np.transpose(y_pred, (*dim_collapse, *dim_final)).reshape(-1, np.prod(shape_final))
+
+        score = r2_score_sklearn(y_true, y_pred, multioutput=multioutput)
+
+        if type(score) == float and np.isnan(score).item():
+            warnings.warn("R2 score is a single NaN, shape matching with NaN")
+            score = np.full(shape_final, np.nan)
+            return score
+
+        if multioutput == 'raw_values':
+            return score.reshape(*shape_final)
+        elif multioutput == 'uniform_average':
+            return score
+        elif multioutput == 'variance_weighted':
+            # return np.mean(score, weights=np.var(y_true, axis=0))
+            raise NotImplementedError("variance_weighted is not implemented yet")
+        else:
+            raise ValueError("multioutput must be one of ['raw_values', 'uniform_average', 'variance_weighted']")
+
+    else:
+        assert (y_true.ndim <= 2) and (y_pred.ndim <= 2), "If axis is None, y_true and y_pred must be smaller than 2D"
+        return r2_score_sklearn(y_true, y_pred, multioutput=multioutput)
+
+# %%

tools/sklearn/model_selection.py

@@ -0,0 +1,229 @@
+# %%
+from sklearn.model_selection import GridSearchCV
+from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit, train_test_split, StratifiedKFold, KFold
+import numpy as np
+import pandas as pd
+import torch
+
+from .. import torch as ttorch
+
+# %%
+'''
+Train/(Validation)/Test Split index functions
+'''
+def stratified_train_test_split_i(y, test_size=0.15, random_state=None):
+    ''':return: indices of train_i, test_i'''
+    x = np.empty(len(y))
+
+    sss = StratifiedShuffleSplit(n_splits=1, test_size=test_size, random_state=random_state)
+    train_i, test_i = next(sss.split(x, y))
+    return train_i, test_i
+
+def stratified_train_val_test_split_i(y, val_size=0.15, test_size=0.15, random_state=None):
+    x = np.empty(len(y))
+
+    sss = StratifiedShuffleSplit(n_splits=1, test_size=test_size, random_state=random_state)
+    train_val_i, test_i = next(sss.split(x, y))
+
+    train_val_y = y[train_val_i]
+    x = np.empty(len(train_val_i))
+    sss = StratifiedShuffleSplit(n_splits=1, test_size=val_size/(1-test_size), random_state=random_state)
+    train_i_, val_i_ = next(sss.split(x, train_val_y))
+    train_i = train_val_i[train_i_]
+    val_i = train_val_i[val_i_]
+
+    return train_i, val_i, test_i
+
+def train_test_split_i(y, test_size=0.15, random_state=None):
+    x = np.empty(len(y))
+
+    ss = ShuffleSplit(n_splits=1, test_size=test_size, random_state=random_state)
+    train_i, test_i = next(ss.split(x, y))
+
+    return train_i, test_i
+
+def train_val_test_split_i(y, val_size=0.15, test_size=0.15, random_state=None):
+    x = np.empty(len(y))
+
+    ss = ShuffleSplit(n_splits=1, test_size=test_size, random_state=random_state)
+    train_val_i, test_i = next(ss.split(x, y))
+
+    train_val_y = y[train_val_i]
+    x = np.empty(len(train_val_i))
+    ss = ShuffleSplit(n_splits=1, test_size=val_size/(1-test_size), random_state=random_state)
+    train_i_, val_i_ = next(ss.split(x, train_val_y))
+    train_i = train_val_i[train_i_]
+    val_i = train_val_i[val_i_]
+
+    return train_i, val_i, test_i
+
+def stratified_kfold_split_i(y, n_splits, split_i, shuffle=True, random_state=None):
+    '''
+    return train, test indices of "split_i"-th split of "n_splits"-fold split
+    '''
+    skf = StratifiedKFold(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
+    x = np.zeros(len(y))
+    skf_ = skf.split(x, y)
+
+    for i in range(split_i):
+        next(skf_)
+    train_i, test_i = next(skf_)
+
+    return train_i, test_i
+
+def stratified_nested_kfold_split_i(y, n_splits, m_splits, split_i, split_j, shuffle=True, random_state=None):
+    '''
+    return train, val, test indices of "split_i"-th split of "n_splits"-fold split
+    '''
+    # Outer split
+    skf = StratifiedKFold(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
+    x = np.zeros(len(y))
+    skf_ = skf.split(x, y)
+
+    for i in range(split_i):
+        next(skf_)
+    train_val_i, test_i = next(skf_) 
+
+    # Inner split
+    skf = StratifiedKFold(n_splits=m_splits, shuffle=shuffle, random_state=random_state)
+    x = np.zeros(len(y[train_val_i]))
+    skf_ = skf.split(x, y[train_val_i])
+
+    for j in range(split_j):
+        next(skf_)
+    train_i, val_i = next(skf_)
+    train_i, val_i = train_val_i[train_i], train_val_i[val_i]
+
+    return train_i, val_i, test_i
+
+def kfold_split_i(y, n_splits, split_i, shuffle=True, random_state=None):
+    '''
+    return train, test indices of "split_i"-th split of "n_splits"-fold split
+    '''
+    kf = KFold(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
+    kf_ = kf.split(y)
+
+    for i in range(split_i):
+        next(kf_)
+    train_i, test_i = next(kf_)
+
+    return train_i, test_i
+
+def nested_kfold_split_i(y, n_splits, m_splits, split_i, split_j, shuffle=True, random_state=None):
+    '''
+    return train, val, test indices of "split_i"-th split of "n_splits"-fold split
+    '''
+    # Outer split
+    kf = KFold(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
+    kf_ = kf.split(y)
+
+    for i in range(split_i):
+        next(kf_)
+    train_val_i, test_i = next(kf_) 
+
+    # Inner split
+    kf = KFold(n_splits=m_splits, shuffle=shuffle, random_state=random_state)
+    kf_ = kf.split(y[train_val_i])
+
+    for j in range(split_j):
+        next(kf_)
+    train_i, val_i = next(kf_)
+    train_i, val_i = train_val_i[train_i], train_val_i[val_i]
+
+    return train_i, val_i, test_i
+
+# %%
+'''
+Train/(Validation)/Test split dataset functions
+'''
+def nested_kfold_split_data(data, y, n_splits, m_splits, split_i, split_j, shuffle=True, random_state=None):
+    '''
+    y doesn't matter since it's not stratified
+    '''
+    train_i, val_i, test_i = nested_kfold_split_i(y, n_splits, m_splits, split_i, split_j, shuffle, random_state)
+    train_data, val_data, test_data = wrap_data(data, train_i), wrap_data(data, val_i), wrap_data(data, test_i)
+    return train_data, val_data, test_data
+    
+def stratified_nested_kfold_split_data(data, y, n_splits, m_splits, split_i, split_j, shuffle=True, random_state=None):
+    """
+    Split data into train, validation, and test set
+
+    Parameters
+    ----------
+    data : pd.DataFrame or np.ndarray or torch.Tensor or torch.dat.Dataset object or iterable object (i.e. list)
+        Data to split
+
+    y: pd.Series or np.ndarray or torch.Tensor or iterable object (i.e. list)
+        Target variable to be used in stratified split
+
+    Returns
+    -------
+    train_data : train data of the same variable format
+    val_data : validation data of the same variable format
+    test_data : test data of the same variable format
+    """
+    train_i, val_i, test_i = stratified_nested_kfold_split_i(y, n_splits, m_splits, split_i, split_j, shuffle, random_state)
+    train_data, val_data, test_data = wrap_data(data, train_i), wrap_data(data, val_i), wrap_data(data, test_i)
+    return train_data, val_data, test_data
+
+def wrap_data(data, i):
+    if isinstance(data, np.ndarray):
+        return data[i]
+    elif isinstance(data, pd.DataFrame):
+        return data.iloc[i]
+    elif isinstance(data, torch.Tensor):
+        return data[i]
+    elif isinstance(data, torch.utils.data.Dataset):
+        return ttorch.data.ProxyDataset(dataset=data, idxs=i)
+    else:
+        return data[i]
+
+# %%
+# def train_val_test_split(x, val_size=0.1, test_size=0.1, random_state=None):
+#     if type(x)==int:
+#         x = np.zeros(x)
+#     ss = ShuffleSplit(n_splits=1, test_size=test_size, random_state=random_state)
+#     train_val_i, test_i = next(ss.split(x))
+#     train_i, val_i = train_test_split(train_val_i, test_size=val_size/(1-test_size), random_state=random_state)
+#
+#     # assert len(set(np.concatenate((train_i, val_i, test_i)))) == len(x)
+#     return train_i, val_i, test_i
+
+class MultiGridSearchCV():
+    '''Perform Grid Search over multiple models
+
+    Parameters
+    ----------
+    estimators: dict
+        Name of model. This name has to correspond to model name of param_grid
+    param_grids: dict
+        Parameter Grid to Search. It must be dict nested in dict(Double dict).
+    **kwargs: other parameters from GridSearchCV
+        Look GridSearchCV
+
+    examples of **kwargs
+    --------------------
+    cv: number of k-fold CV
+    scoring: scoring method. either can be prebuilt sklearn string or a callable function
+
+    '''
+    def __init__(self, estimators, param_grids, **kwargs):
+        self.estimators = estimators
+        self.param_grids = param_grids
+        self.kwargs = kwargs
+        self.grid_searches = {estimator_name: GridSearchCV(estimator=estimator, param_grid=self.param_grids[estimator_name], **self.kwargs) for estimator_name, estimator in self.estimators.items()}
+
+    def fit(self, x, y):
+        for grid_search in self.grid_searches.values():
+            grid_search.fit(x,y)
+        self.cv_results = {estimator_name: grid_search.cv_results_ for estimator_name, grid_search in self.grid_searches.items()}
+        best_estimator_name_, best_grid_search_ = max(self.grid_searches.items(), key = lambda x: x[1].best_score_) # x[0]: key, x[1]: value
+        self.best_score_ = best_grid_search_.best_score_
+        self.best_estimator_ = best_grid_search_.best_estimator_
+        self.best_params_ = best_grid_search_.best_params_
+        self.best_estimator_name_ = best_estimator_name_
+
+    def predict(self, x):
+        assert hasattr(self, 'best_estimator_'), 'There is no best_estimator_. Need to call "fit" first.'
+        assert hasattr(self.best_estimator_,'predict'), 'Best estimator does not support "predict" method.'
+        return self.best_estimator_.predict(x)

tools/sklearn/preprocessing.py

@@ -0,0 +1,125 @@
+from sklearn.preprocessing import StandardScaler, MinMaxScaler
+from sklearn.decomposition import PCA
+import numpy as np
+
+def standardize(x, scaler=None):
+    """
+    Standardize (z-score/gaussian normalization) x without having to make a StandardScaler() object.
+
+    Parameters
+    ----------
+    x : ndarray of shape (n_observation, n_channel) 
+        data to be standardized.
+    scaler: sklearn.preprocessing.StandardScaler(), default=None
+        scaler to use (that has already fit())
+
+    Returns
+    -------
+    x_standardized : ndarray of shape (n_observation, n_channel)
+        Normalized data
+    scaler : sklearn.preprocessing.StandardScaler()
+        Scaler used for normalization
+    """
+    if scaler is None:
+        scaler = StandardScaler()
+        x_standardized = scaler.fit_transform(x)
+    else:
+        x_standardized = scaler.transform(x)
+    
+    return x_standardized, scaler
+
+def project_pca(x, var_threshold=None, n_pc=None, solver=None, random_state=0):
+    """
+    project x onto PC space without having to make a PCA() object.
+
+    Parameters
+    ----------
+    x : ndarray of shape (n_observation, n_channel)
+        data to be projected.
+    solver: sklearn.decomposition.PCA(), default=None
+        solver to use (that has already fit())
+
+    Returns
+    -------
+    x_projected : ndarray of shape (n_observation, n_channel)
+        Normalized data
+    solver : sklearn.decomposition.PCA()
+        Solver used for PCA projection
+    """
+    # If svd_solver is not full, result is random
+    if solver is None:
+        solver = PCA(svd_solver='full', random_state=random_state)
+        x_pc = solver.fit_transform(x)
+    else:
+        x_pc = solver.transform(x)
+
+    if var_threshold is not None:
+        n_pc = np.argmax(np.cumsum(solver.explained_variance_ratio_) > var_threshold) + 1
+    elif n_pc is not None:
+        pass
+    else:
+        n_pc = solver.n_components_
+    x_pc = x_pc[:,:n_pc]
+
+    return x_pc, solver
+
+# def minmaxscale(x, scaler=None, feature_range=(0, 1), *, copy=True, clip=False):
+def minmaxscale(x, scaler=None, **kwargs):
+    """
+    Min-max scale x without having to make a MinMaxScaler() object.
+
+    Parameters
+    ----------
+    x : ndarray of shape (n_observation, n_channel) 
+        data to be standardized.
+    scaler: sklearn.preprocessing.MinMaxScaler(), default=None
+        scaler to use (that has already fit())
+
+
+    Following arguments are applied if scaler is None:
+
+    feature_range : tuple (min, max), default=(0, 1)
+        Desired range of transformed data.
+        The default is (0, 1).
+    copy : bool, default=True
+        Set to False to perform inplace normalization and avoid a copy (if the input is already a numpy array).
+    clip : bool, default=False
+        Set to True to clip transformed values of held-out data.
+
+    Returns
+    -------
+    x_scaled : ndarray of shape (n_observation, n_channel)
+        Normalized data
+    scaler : sklearn.preprocessing.MinMaxScaler()
+        Scaler used for normalization
+    """
+    if scaler is None:
+        # scaler = MinMaxScaler(feature_range=feature_range, copy=copy, clip=clip)
+        scaler = MinMaxScaler(**kwargs)
+        x_scaled = scaler.fit_transform(x)
+    else:
+        x_scaled = scaler.transform(x)
+    
+    return x_scaled, scaler
+
+if __name__=='__main__':
+    import tools as T
+
+    x = np.random.rand(2,3,5)
+    print(x.shape)
+    squeezer = T.numpy.Squeezer()
+    x = squeezer.squeeze(x)
+    print(x.shape)
+    x_s, scaler = T.sklearn.preprocessing.standardize(x)
+    x_s = squeezer.unsqueeze(x_s)
+    print(x_s.shape)
+
+    # %%
+    x = np.random.rand(2,3,5)
+    print(x.shape)
+    squeezer = T.numpy.Squeezer()
+    x = squeezer.squeeze(x)
+    print(x.shape)
+    x_pc, solver = T.sklearn.preprocessing.project_pca(x, n_pc=3)
+    x_pc = squeezer.unsqueeze(x_pc, strict=False)
+    print(x_pc.shape)

tools/stats/__init__.py

@@ -0,0 +1,43 @@
+import numpy as np
+import scipy.stats as st
+
+def ci(std, n, p=0.95):
+    z = st.norm.ppf((1+p)/2) # Two-tailed confidence interval
+    half_range = z*std/np.sqrt(n)
+    return half_range
+
+def interval(distrib, confidence, **kwargs):
+    if 'std' in kwargs and 'n' in kwargs:
+        scale = kwargs['std']/np.sqrt(kwargs['n'])
+        assert 'scale' not in kwargs, 'scale is given but std and n are also given'
+        kwargs['scale'] = scale
+        del kwargs['std'], kwargs['n']
+
+    return distrib.interval(confidence, **kwargs)
+
+def ci_stats(mean, std, n, p=0.95):
+    '''
+    calculate confidence interval based on given statistics
+    :param mean: 1d array, sample mean
+    :param std: 1d array, sample mean
+    :param n: 1d array, sample size
+    :param p: 1d array, confidence level
+    '''
+    # z = st.norm.ppf(p)
+    # half_range = z*std/n
+    half_range = ci(std=std, n=n)
+
+    confidence_interval = {
+    'low': mean-half_range,
+    'high': mean+half_range
+    }
+    return confidence_interval
+
+def ci_samples(x, p=0.95):
+    '''
+    calculate confidence interval based on samples
+    '''
+    mean = np.mean(x)
+    std = np.std(x)
+    n = len(x)
+    return ci_stats(mean, std, n, p=p)