cpgtools 2.0.0__py3-none-any.whl → 2.0.2__py3-none-any.whl

impyute/dataset/base.py

@@ -0,0 +1,137 @@
+""" Shared functions to load/generate data """
+import itertools
+import math
+import random
+import string
+import numpy as np
+from impyute.dataset.corrupt import Corruptor
+from impyute.ops import error
+
+def randu(bound=(0, 10), shape=(5, 5), missingness="mcar", thr=0.2, dtype="int"):
+    """ Return randomly generated dataset of numbers with uniformly
+    distributed values between bound[0] and bound[1]
+
+    Parameters
+    ----------
+    bound:tuple (start,stop)
+        Determines the range of values in the matrix. Index 0 for start
+        value and index 1 for stop value. Start is inclusive, stop is
+        exclusive.
+    shape:tuple(optional)
+        Size of the randomly generated data
+    missingness: ('mcar', 'mar', 'mnar')
+        Type of missingness you want in your dataset
+    thr: float between [0,1]
+        Percentage of missing data in generated data
+    dtype: ('int','float')
+        Type of data
+
+    Returns
+    -------
+    numpy.ndarray
+    """
+    if dtype == "int":
+        data = np.random.randint(bound[0], bound[1], size=shape).astype(float)
+    elif dtype == "float":
+        data = np.random.uniform(bound[0], bound[1], size=shape)
+    corruptor = Corruptor(data, thr=thr)
+    raw_data = getattr(corruptor, missingness)()
+    return raw_data
+
+
+def randn(theta=(0, 1), shape=(5, 5), missingness="mcar", thr=0.2, dtype="float"):
+    """ Return randomly generated dataset of numbers with normally
+    distributed values with given and sigma.
+
+    Parameters
+    ----------
+    theta: tuple (mu, sigma)
+        Determines the range of values in the matrix
+    shape:tuple(optional)
+        Size of the randomly generated data
+    missingness: ('mcar', 'mar', 'mnar')
+        Type of missingness you want in your dataset
+    thr: float between [0,1]
+        Percentage of missing data in generated data
+    dtype: ('int','float')
+        Type of data
+
+    Returns
+    -------
+    numpy.ndarray
+    """
+    mean, sigma = theta
+    data = np.random.normal(mean, sigma, size=shape)
+    if dtype == "int":
+        data = np.round(data)
+    elif dtype == "float":
+        pass
+    corruptor = Corruptor(data, thr=thr)
+    raw_data = getattr(corruptor, missingness)()
+    return raw_data
+
+def randc(nlevels=5, shape=(5, 5), missingness="mcar", thr=0.2):
+    """ Return randomly generated dataset with uniformly distributed categorical data (alphabetic character)
+
+    Parameters
+    ----------
+    nlevels: int
+        Specify the number of different categories in the dataset
+    shape: tuple(optional)
+        Size of the randomly generated data
+    missingness: string in ('mcar', 'mar', 'mnar')
+        Type of missingness you want in your dataset
+    thr: float between [0,1]
+        Percentage of missing data in generated data
+
+    Returns
+    -------
+    numpy.ndarray
+    """
+    if shape[0]*shape[1] < nlevels:
+        raise error.BadInputError("nlevel exceeds the size of desired dataset. Please decrease the nlevel or increase the shape")
+
+    length = len(string.ascii_lowercase)
+    n_fold = int(math.floor(math.log(nlevels, length)))
+    cat_pool = list(string.ascii_lowercase)
+
+    # when nlevel > 26, the alphabetical character is used up, need to generate extra strings as categorical data
+    if n_fold > 0:
+        for i in range(2, n_fold+2):
+            pool_candidate = list(itertools.product(string.ascii_lowercase, repeat=i))
+            cat_pool.extend([''.join(w) for w in pool_candidate])
+            if len(cat_pool) > nlevels:
+                break
+
+    cat = random.sample(cat_pool, nlevels)
+    data = np.random.choice(cat, shape, replace=True)
+
+    # make sure the data frame has nlevel different categories
+    while len(np.unique(data)) != nlevels:
+        data = np.random.choice(cat, shape, replace=True)
+
+    corruptor = Corruptor(data, thr=thr, dtype=np.str)
+    raw_data = getattr(corruptor, missingness)()
+    return raw_data
+
+
+
+def mnist(missingness="mcar", thr=0.2):
+    """ Loads corrupted MNIST
+
+    Parameters
+    ----------
+    missingness: ('mcar', 'mar', 'mnar')
+        Type of missigness you want in your dataset
+    th: float between [0,1]
+        Percentage of missing data in generated data
+
+    Returns
+    -------
+    numpy.ndarray
+    """
+    from sklearn.datasets import fetch_mldata
+    dataset = fetch_mldata('MNIST original')
+    corruptor = Corruptor(dataset.data, thr=thr)
+    data = getattr(corruptor, missingness)()
+    return {"X": data, "Y": dataset.target}

impyute/dataset/corrupt.py

@@ -0,0 +1,55 @@
+""" impyute.dataset.corrupt """
+import numpy as np
+
+
+class Corruptor:
+    """ Adds missing values to a complete dataset.
+
+    Attributes
+    ----------
+    data: np.ndarray
+        Matrix of values with no NaN's that you want to add NaN's to.
+    thr: float (optional)
+        The percentage of null values you want in your dataset, a number
+        between 0 and 1.
+
+    Methods
+    -------
+    mcar()
+        Overwrite values with MCAR placed NaN's.
+    mar()
+        Overwrite values with MAR placed NaN's.
+    mnar()
+        Overwrite values with MNAR placed NaN's.
+
+    """
+    def __init__(self, data, thr=0.2, dtype=np.float):
+        self.dtype = data.dtype
+        self.shape = np.shape(data)
+        self.data = data.astype(dtype)
+        self.thr = thr
+
+    def mcar(self):
+        """ Overwrites values with MCAR placed NaN's """
+        data_1d = self.data.flatten()
+        n_total = len(data_1d)
+        nan_x = np.random.choice(range(n_total),
+                                  size=int(self.thr*n_total),
+                                  replace=False)
+        for x_i in nan_x:
+            data_1d[x_i] = np.nan
+        output = data_1d.reshape(self.shape)
+        return output
+
+    def mar(self):
+        """ Overwrites values with MAR placed NaN's """
+        pass
+
+    def mnar(self):
+        """ Overwrites values with MNAR placed NaN's """
+        pass
+
+    def complete(self):
+        """ Do nothing to the data """
+        output = self.data
+        return output

impyute/deletion/__init__.py

@@ -0,0 +1,5 @@
+""" Missing data approaches that delete values.  """
+
+from .complete_case import complete_case
+
+__all__ = ["complete_case"]

impyute/deletion/complete_case.py

@@ -0,0 +1,21 @@
+""" impyute.deletion.complete_case """
+import numpy as np
+from impyute.ops import wrapper
+
+@wrapper.wrappers
+@wrapper.checks
+def complete_case(data):
+    """ Return only data rows with all columns
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        Data to impute.
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    """
+    return data[~np.isnan(data).any(axis=1)]

impyute/ops/__init__.py

@@ -0,0 +1,12 @@
+""" Unorganized set of utility functions """
+
+from . import error
+from . import inverse_distance_weighting
+from . import matrix
+from . import util
+from . import wrapper
+
+__all__ = [
+        "error", "inverse_distance_weighting", "matrix",
+        "util", "wrapper"
+        ]

impyute/ops/error.py

@@ -0,0 +1,9 @@
+""" Impyute specific error messages """
+
+class BadInputError(Exception):
+    "Error thrown when input args don't match spec"
+    pass
+
+class BadOutputError(Exception):
+    "Error thrown when outputs don't match spec"
+    pass

impyute/ops/inverse_distance_weighting.py

@@ -0,0 +1,31 @@
+""" Assign weights to distances in a way such that farther values are weighed less """
+import numpy as np
+
+def shepards(distances, power=2):
+    """ Basic inverse distance weighting function
+
+    Parameters
+    ----------
+    distances: list/numpy.ndarray
+        1D list of numbers (ex. distance results from call to KDTree.query)
+
+    power: int
+        Default of 2 used since the referenced paper stated an exponent of 2 "gives seemingly
+        satisfactory results"
+
+    Returns
+    -------
+    numpy.ndarray
+        1D list of numbers that sum to 1, represents weights of provided distances, in order.
+
+    References
+    ----------
+
+    Shepard, Donald (1968). "A two-dimensional interpolation function for irregularly-spaced data".
+    Proceedings of the 1968 ACM National Conference. pp. 517-524. doi:10.1145/800186.810616
+    """
+    return to_percentage(1/np.power(distances, power))
+
+def to_percentage(vec):
+    """ Converts list of real numbers into a list of percentages """
+    return vec/np.sum(vec)

impyute/ops/matrix.py

@@ -0,0 +1,47 @@
+""" Common operations on matrices
+
+*Look into whether it's worth writing these in raw c*
+"""
+import numpy as np
+
+def nan_indices(data):
+    """ Finds the indices of all missing values.
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+
+    Returns
+    -------
+    List of tuples
+        Indices of all missing values in tuple format; (i, j)
+    """
+    return np.argwhere(np.isnan(data))
+
+def map_nd(fn, arr):
+    """ Map fn that takes a value over entire n-dim array
+
+    Parameters
+    ----------
+    arr: numpy.ndarray
+
+    Returns
+    -------
+    numpy.ndarray
+
+    """
+    return np.vectorize(fn)(arr)
+
+def every_nd(fn, arr):
+    """ Returns bool, true if fn is true for all elements of arr
+
+    Parameters
+    ----------
+    arr: numpy.ndarray
+
+    Returns
+    -------
+    bool
+
+    """
+    return all(map(fn, arr.flatten()))

impyute/ops/testing.py

@@ -0,0 +1,20 @@
+""" Utilities used for unit tests """
+import numpy as np
+
+
+def return_na_check(data):
+    """Helper function for tests to check if the data returned is a
+       numpy array and that the imputed data has no NaN's.
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        Data to impute.
+    
+    Returns
+    -------
+    None
+
+    """
+    assert isinstance(data, np.ndarray)
+    assert not np.isnan(data).any()

impyute/ops/util.py

@@ -0,0 +1,76 @@
+""" Random utility functions """
+from functools import wraps
+import numpy as np
+import pandas as pd
+
+# Things that get exposed from * import
+__all__ = [
+    "constantly", "complement", "identity", "thread",
+    "execute_fn_with_args_and_or_kwargs", "toy_df",
+    "insert_na",
+    ]
+
+def thread(arg, *fns):
+    if len(fns) > 0:
+        return thread(fns[0](arg), *fns[1:])
+    else:
+        return arg
+
+def identity(x):
+    return x
+
+def constantly(x):
+    """ Returns a function that takes any args and returns x """
+    def func(*args, **kwargs):
+        return x
+    return func
+
+def complement(fn):
+    """ Return fn that outputs the opposite truth values of the
+    input function
+    """
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        return not fn(*args, **kwargs)
+    return wrapper
+
+def execute_fn_with_args_and_or_kwargs(fn, args, kwargs):
+    """ If args + kwargs aren't accepted only args are passed in"""
+    try:
+        return fn(*args, **kwargs)
+    except TypeError:
+        return fn(*args)
+
+def toy_df(nrow, ncol, n_miss, sample_prefix, seed):
+    """
+    Make a dataFrame (nrow x ncol) with random values between 0 and 1, add
+    some missing values (n_miss). Generate a toy dataframe for testing purposes.
+    """
+    np.random.seed(seed)
+    data = np.random.rand(nrow*ncol).reshape((nrow, ncol)).astype(float)
+    x_ind = np.random.choice(nrow, n_miss)
+    y_ind = np.random.choice(ncol, n_miss)
+    for x,y in zip(x_ind, y_ind):
+        data[x][y] =  np.nan
+    colNames = [sample_prefix + '_' + str(i) for i in range(0,ncol)]
+    df = pd.DataFrame(data, columns=colNames)
+    return df
+
+def insert_na(df, n_miss, seed):
+    np.random.seed(seed)
+    nrow,ncol = df.shape
+    na_count = 0
+    if n_miss >= nrow*ncol:
+        out_df = df.replace(df.values, np.nan)
+    else:
+        tmp = df.to_numpy()
+        while(1):
+            if na_count >= n_miss:
+                break
+            x_ind = np.random.choice(nrow)
+            y_ind = np.random.choice(ncol)
+            if not np.isnan(tmp[x_ind][y_ind]):
+                tmp[x_ind][y_ind] = np.nan
+                na_count += 1
+        out_df = pd.DataFrame(tmp, index=df.index, columns=df.columns)
+    return out_df

impyute/ops/wrapper.py

@@ -0,0 +1,179 @@
+""" Decorator functions to wrap around entry and exit
+
+... to easily apply to a function, functions that check/process inputs
+and outputs
+"""
+from functools import wraps
+import numpy as np
+import pandas as pd
+
+from . import error
+from . import matrix
+from . import util as u
+
+## Hacky way to handle python2 not having `ModuleNotFoundError`
+# pylint: disable=redefined-builtin, missing-docstring
+try:
+    raise ModuleNotFoundError
+except NameError:
+    class ModuleNotFoundError(Exception):
+        pass
+except ModuleNotFoundError:
+    pass
+# pylint: enable=redefined-builtin, missing-docstring
+
+
+def handle_df(fn):
+    """ Decorator to handle pandas Dataframe object as input
+
+    If the first arg is a pandas dataframe, convert it to a numpy array
+    otherwise don't do anything. Cast back to a pandas Dataframe after
+    the imputation function has run
+    """
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        is_df = False
+        ## convert tuple to list so args can be modified
+        args = list(args)
+        ## Either make a copy or use a pointer to the original
+        if kwargs.get('inplace'):
+            args[0] = args[0]
+        else:
+            args[0] = args[0].copy()
+
+        ## If input data is a dataframe then cast the input to an np.array
+        ## and set an indicator flag before continuing
+        if isinstance(args[0], pd.DataFrame):
+            is_df = True
+            in_ind = args[0].index
+            in_columns = args[0].columns
+            args[0] = args[0].to_numpy()
+
+        ## function invokation
+        results = u.execute_fn_with_args_and_or_kwargs(fn, args, kwargs)
+
+        ## cast the output back to a DataFrame.
+        if is_df:
+            results = pd.DataFrame(results, index=in_ind, columns=in_columns)
+        return results
+    return wrapper
+
+def add_inplace_option(fn):
+    """ Decorator for inplace option
+
+    Functions wrapped by this can have an `inplace` kwarg to use either a copy of
+    data or reference """
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        """ Run input checks"""
+        ## convert tuple to list so args can be modified
+        args = list(args)
+        ## Either make a copy or use a pointer to the original
+        if kwargs.get('inplace'):
+            args[0] = args[0]
+        else:
+            args[0] = args[0].copy()
+
+        ## function invokation
+        return u.execute_fn_with_args_and_or_kwargs(fn, args, kwargs)
+    return wrapper
+
+def conform_output(fn):
+    """ Decorator to handle impossible values
+
+    Adds two optional kwargs, `coerce_fn` and `valid_fn`.
+
+    `valid_fn` function stub
+
+        def my_coerce_fn(some_literal) -> boolean
+
+    `coerce_fn` function stub
+
+        def my_coerce_fn(arr, x_i, y_i) -> some_literal
+
+    Valid function is something run on each element of the, this is
+    the function that we use to indicate whether the value is valid
+    or not
+
+    Coerce function has three arguments, the original matrix and
+    the two indices of the invalid value x_i and y_i. This function
+    will be run on all invalid values.
+    """
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        def raise_error(arr, x_i, y_i):
+            raise error.BadOutputError("{} does not conform".format(arr[x_i, y_i]))
+        ## convert tuple to list so args can be modified
+        args = list(args)
+        # function that checks if the value is valid
+        valid_fn = kwargs.get("valid_fn", u.constantly(True))
+        # function that modifies the invalid value to something valid
+        coerce_fn = kwargs.get("coerce_fn", raise_error)
+
+        ## function invokation
+        results = u.execute_fn_with_args_and_or_kwargs(fn, args, kwargs)
+
+        # check each value to see if it's valid
+        bool_arr = matrix.map_nd(u.complement(valid_fn), results)
+        # get indices of invalid values
+        invalid_indices = np.argwhere(bool_arr)
+        # run the coerce fn on each invalid indice
+        for x_i, y_i in invalid_indices:
+            results[x_i, y_i] = coerce_fn(results, x_i, y_i)
+
+        return results
+    return wrapper
+
+def wrappers(fn):
+    """ Helper decorator, all wrapper functions applied to modify input (matrix
+    with missing values) and output (matrix with imputed values)
+
+    NOTE: `handle_df` has to be last as it needs to be in the outer loop (first
+    entry point) since every other function assumes you're getting an np.array
+    as input
+    """
+    return u.thread(
+        fn,                 # function that's getting wrapped
+        add_inplace_option, # allow choosing reference/copy
+        conform_output,     # allow enforcing of some spec on returned outputs
+        handle_df,          # if df type, cast to np.array on in and df on out
+    )
+
+def _shape_2d(data):
+    """ True if array is 2D"""
+    return len(np.shape(data)) == 2
+
+def _shape_3d(data):
+    """ True if array is 3D"""
+    return len(np.shape(data)) == 3
+
+def _is_ndarray(data):
+    """ True if the array is an instance of numpy's ndarray"""
+    return isinstance(data, np.ndarray)
+
+def _dtype_float(data):
+    """ True if the values in the array are floating point"""
+    return data.dtype == float
+
+def _nan_exists(data):
+    """ True if there is at least one np.nan in the array"""
+    nan_xy = matrix.nan_indices(data)
+    return len(nan_xy) > 0
+
+def checks(fn):
+    """ Throw exception if error runs"""
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        data = args[0]
+        if len(np.shape(data)) != 2:
+            raise error.BadInputError("No support for arrays that aren't 2D yet.")
+        elif not _shape_2d(data):
+            raise error.BadInputError("Not a 2D array.")
+        elif not _is_ndarray(data):
+            raise error.BadInputError("Not a np.ndarray.")
+        elif not _dtype_float(data):
+            raise error.BadInputError("Data is not float.")
+        elif not _nan_exists(data):
+            raise error.BadInputError("No NaN's in given data")
+        return u.execute_fn_with_args_and_or_kwargs(fn, args, kwargs)
+    return wrapper

impyute/ts/__init__.py

@@ -0,0 +1,6 @@
+""" Imputations for time-series data.  """
+
+from .locf import locf
+from .moving_window import moving_window
+
+__all__ = ["locf", "moving_window"]

impyute/ts/locf.py

@@ -0,0 +1,57 @@
+import numpy as np
+from impyute.ops import matrix
+from impyute.ops import wrapper
+from impyute.ops import error
+
+@wrapper.wrappers
+@wrapper.checks
+def locf(data, axis=0):
+    """ Last Observation Carried Forward
+
+    For each set of missing indices, use the value of one row before(same
+    column). In the case that the missing value is the first row, look one
+    row ahead instead. If this next row is also NaN, look to the next row.
+    Repeat until you find a row in this column that's not NaN. All the rows
+    before will be filled with this value.
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        Data to impute.
+    axis: boolean (optional)
+        0 if time series is in row format (Ex. data[0][:] is 1st data point).
+        1 if time series is in col format (Ex. data[:][0] is 1st data point).
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    """
+    if axis == 0:
+        data = np.transpose(data)
+    elif axis == 1:
+        pass
+    else:
+        raise error.BadInputError("Error: Axis value is invalid, please use either 0 (row format) or 1 (column format)")
+
+    nan_xy = matrix.nan_indices(data)
+    for x_i, y_i in nan_xy:
+        # Simplest scenario, look one row back
+        if x_i-1 > -1:
+            data[x_i][y_i] = data[x_i-1][y_i]
+        # Look n rows forward
+        else:
+            x_residuals = np.shape(data)[0]-x_i-1  # n datapoints left
+            val_found = False
+            for i in range(1, x_residuals):
+                if not np.isnan(data[x_i+i][y_i]):
+                    val_found = True
+                    break
+            if val_found:
+                # pylint: disable=undefined-loop-variable
+                for x_nan in range(i):
+                    data[x_i+x_nan][y_i] = data[x_i+i][y_i]
+            else:
+                raise Exception("Error: Entire Column is NaN")
+    return data